Skip to content

Commit 5b51ddc

Browse files
joegallomasseyke
joegallo
and
masseyke
authored
[8.19] Backport data stream settings (#129213)
Co-authored-by: Keith Massey <[email protected]>
1 parent f02d15c commit 5b51ddc

File tree

41 files changed

+3027
-52
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+3027
-52
lines changed

‎docs/reference/data-streams/change-mappings-and-settings.asciidoc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -596,7 +596,8 @@ stream's oldest backing index.
596596
"allow_custom_routing": false,
597597
"replicated": false,
598598
"index_mode": "standard",
599-
"rollover_on_write": false
599+
"rollover_on_write": false,
600+
"settings": {}
600601
}
601602
]
602603
}

‎docs/reference/data-streams/downsampling-manual.asciidoc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -382,7 +382,8 @@ This returns:
382382
"end": "2023-07-26T13:26:42.000Z"
383383
}
384384
]
385-
}
385+
},
386+
"settings": {}
386387
}
387388
]
388389
}

‎docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,14 @@ being managed by data stream lifecycle. As we'll see, {ilm-init} and data stream
1111
can co-manage a data stream; however, an index can only be managed by one system at
1212
a time.
1313

14+
[source,console]
15+
--------------------------------------------------
16+
DELETE _data_stream/dsl-data-stream
17+
DELETE _index_template/dsl-data-stream-template
18+
DELETE _ilm/policy/pre-dsl-ilm-policy
19+
--------------------------------------------------
20+
// TEARDOWN
21+
1422
[discrete]
1523
[[migrate-dsl-ilm-tldr]]
1624
==== TL;DR
@@ -141,7 +149,8 @@ and that the next generation index will also be managed by {ilm-init}:
141149
"allow_custom_routing": false,
142150
"replicated": false,
143151
"index_mode": "standard",
144-
"rollover_on_write": false
152+
"rollover_on_write": false,
153+
"settings": {}
145154
}
146155
]
147156
}
@@ -283,7 +292,8 @@ GET _data_stream/dsl-data-stream
283292
"allow_custom_routing": false,
284293
"replicated": false,
285294
"index_mode": "standard",
286-
"rollover_on_write": false
295+
"rollover_on_write": false,
296+
"settings": {}
287297
}
288298
]
289299
}
@@ -367,7 +377,8 @@ GET _data_stream/dsl-data-stream
367377
"allow_custom_routing": false,
368378
"replicated": false,
369379
"index_mode": "standard",
370-
"rollover_on_write": false
380+
"rollover_on_write": false,
381+
"settings": {}
371382
}
372383
]
373384
}
@@ -469,7 +480,8 @@ GET _data_stream/dsl-data-stream
469480
"allow_custom_routing": false,
470481
"replicated": false,
471482
"index_mode": "standard",
472-
"rollover_on_write": false
483+
"rollover_on_write": false,
484+
"settings": {}
473485
}
474486
]
475487
}
@@ -488,15 +500,3 @@ GET _data_stream/dsl-data-stream
488500
Had we removed the {ilm-init} policy from the index template when we <<update-index-template-for-dsl, updated>>
489501
it, the write index of the data stream will now be `Unmanaged` because the index
490502
wouldn't have the {ilm-init} policy configured to fallback onto.
491-
492-
//////////////////////////
493-
[source,console]
494-
--------------------------------------------------
495-
DELETE _data_stream/dsl-data-stream
496-
DELETE _index_template/dsl-data-stream-template
497-
DELETE _ilm/policy/pre-dsl-ilm-policy
498-
--------------------------------------------------
499-
// TEST[continued]
500-
501-
//////////////////////////
502-

‎docs/reference/indices/get-data-stream.asciidoc‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,8 @@ The API returns the following response:
330330
"system": false,
331331
"allow_custom_routing": false,
332332
"replicated": false,
333-
"rollover_on_write": false
333+
"rollover_on_write": false,
334+
"settings": {}
334335
},
335336
{
336337
"name": "my-data-stream-two",
@@ -361,7 +362,8 @@ The API returns the following response:
361362
"system": false,
362363
"allow_custom_routing": false,
363364
"replicated": false,
364-
"rollover_on_write": false
365+
"rollover_on_write": false,
366+
"settings": {}
365367
}
366368
]
367369
}

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java‎

Lines changed: 351 additions & 0 deletions
Large diffs are not rendered by default.

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ public class DataStreamFeatures implements FeatureSpecification {
3333
"data_stream.downsample.default_aggregate_metric_fix"
3434
);
3535

36+
public static final NodeFeature LOGS_STREAM_FEATURE = new NodeFeature("logs_stream");
37+
3638
@Override
3739
public Map<NodeFeature, Version> getHistoricalFeatures() {
3840
return Map.of(DATA_STREAM_LIFECYCLE, Version.V_8_11_0);
@@ -51,6 +53,6 @@ public Set<NodeFeature> getFeatures() {
5153

5254
@Override
5355
public Set<NodeFeature> getTestFeatures() {
54-
return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX);
56+
return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX, LOGS_STREAM_FEATURE);
5557
}
5658
}

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@
1616
import org.elasticsearch.action.datastreams.DataStreamsStatsAction;
1717
import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
1818
import org.elasticsearch.action.datastreams.GetDataStreamAction;
19+
import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction;
1920
import org.elasticsearch.action.datastreams.MigrateToDataStreamAction;
2021
import org.elasticsearch.action.datastreams.ModifyDataStreamsAction;
2122
import org.elasticsearch.action.datastreams.PromoteDataStreamAction;
2223
import org.elasticsearch.action.datastreams.PutDataStreamOptionsAction;
24+
import org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction;
2325
import org.elasticsearch.action.datastreams.lifecycle.ExplainDataStreamLifecycleAction;
2426
import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction;
2527
import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction;
@@ -37,10 +39,12 @@
3739
import org.elasticsearch.datastreams.action.TransportCreateDataStreamAction;
3840
import org.elasticsearch.datastreams.action.TransportDataStreamsStatsAction;
3941
import org.elasticsearch.datastreams.action.TransportDeleteDataStreamAction;
42+
import org.elasticsearch.datastreams.action.TransportGetDataStreamSettingsAction;
4043
import org.elasticsearch.datastreams.action.TransportGetDataStreamsAction;
4144
import org.elasticsearch.datastreams.action.TransportMigrateToDataStreamAction;
4245
import org.elasticsearch.datastreams.action.TransportModifyDataStreamsAction;
4346
import org.elasticsearch.datastreams.action.TransportPromoteDataStreamAction;
47+
import org.elasticsearch.datastreams.action.TransportUpdateDataStreamSettingsAction;
4448
import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore;
4549
import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService;
4650
import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamLifecycleAction;
@@ -68,10 +72,12 @@
6872
import org.elasticsearch.datastreams.rest.RestCreateDataStreamAction;
6973
import org.elasticsearch.datastreams.rest.RestDataStreamsStatsAction;
7074
import org.elasticsearch.datastreams.rest.RestDeleteDataStreamAction;
75+
import org.elasticsearch.datastreams.rest.RestGetDataStreamSettingsAction;
7176
import org.elasticsearch.datastreams.rest.RestGetDataStreamsAction;
7277
import org.elasticsearch.datastreams.rest.RestMigrateToDataStreamAction;
7378
import org.elasticsearch.datastreams.rest.RestModifyDataStreamsAction;
7479
import org.elasticsearch.datastreams.rest.RestPromoteDataStreamAction;
80+
import org.elasticsearch.datastreams.rest.RestUpdateDataStreamSettingsAction;
7581
import org.elasticsearch.features.NodeFeature;
7682
import org.elasticsearch.health.HealthIndicatorService;
7783
import org.elasticsearch.index.IndexSettingProvider;
@@ -241,6 +247,8 @@ public Collection<?> createComponents(PluginServices services) {
241247
actions.add(new ActionHandler<>(GetDataStreamOptionsAction.INSTANCE, TransportGetDataStreamOptionsAction.class));
242248
actions.add(new ActionHandler<>(PutDataStreamOptionsAction.INSTANCE, TransportPutDataStreamOptionsAction.class));
243249
actions.add(new ActionHandler<>(DeleteDataStreamOptionsAction.INSTANCE, TransportDeleteDataStreamOptionsAction.class));
250+
actions.add(new ActionHandler<>(GetDataStreamSettingsAction.INSTANCE, TransportGetDataStreamSettingsAction.class));
251+
actions.add(new ActionHandler<>(UpdateDataStreamSettingsAction.INSTANCE, TransportUpdateDataStreamSettingsAction.class));
244252
return actions;
245253
}
246254

@@ -276,6 +284,8 @@ public List<RestHandler> getRestHandlers(
276284
handlers.add(new RestGetDataStreamOptionsAction());
277285
handlers.add(new RestPutDataStreamOptionsAction());
278286
handlers.add(new RestDeleteDataStreamOptionsAction());
287+
handlers.add(new RestGetDataStreamSettingsAction());
288+
handlers.add(new RestUpdateDataStreamSettingsAction());
279289
return handlers;
280290
}
281291

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.datastreams.action;
11+
12+
import org.elasticsearch.action.ActionListener;
13+
import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction;
14+
import org.elasticsearch.action.support.ActionFilters;
15+
import org.elasticsearch.action.support.IndicesOptions;
16+
import org.elasticsearch.action.support.TransportLocalClusterStateAction;
17+
import org.elasticsearch.cluster.ClusterState;
18+
import org.elasticsearch.cluster.block.ClusterBlockException;
19+
import org.elasticsearch.cluster.block.ClusterBlockLevel;
20+
import org.elasticsearch.cluster.metadata.DataStream;
21+
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
22+
import org.elasticsearch.cluster.service.ClusterService;
23+
import org.elasticsearch.common.settings.Settings;
24+
import org.elasticsearch.common.settings.SettingsFilter;
25+
import org.elasticsearch.injection.guice.Inject;
26+
import org.elasticsearch.tasks.Task;
27+
import org.elasticsearch.threadpool.ThreadPool;
28+
import org.elasticsearch.transport.TransportService;
29+
30+
import java.util.ArrayList;
31+
import java.util.List;
32+
import java.util.Map;
33+
34+
public class TransportGetDataStreamSettingsAction extends TransportLocalClusterStateAction<
35+
GetDataStreamSettingsAction.Request,
36+
GetDataStreamSettingsAction.Response> {
37+
private final IndexNameExpressionResolver indexNameExpressionResolver;
38+
private final SettingsFilter settingsFilter;
39+
40+
@Inject
41+
public TransportGetDataStreamSettingsAction(
42+
TransportService transportService,
43+
ClusterService clusterService,
44+
ThreadPool threadPool,
45+
SettingsFilter settingsFilter,
46+
ActionFilters actionFilters,
47+
IndexNameExpressionResolver indexNameExpressionResolver
48+
) {
49+
super(
50+
GetDataStreamSettingsAction.NAME,
51+
clusterService,
52+
transportService,
53+
actionFilters,
54+
GetDataStreamSettingsAction.Request::localOnly,
55+
threadPool.executor(ThreadPool.Names.MANAGEMENT)
56+
);
57+
this.indexNameExpressionResolver = indexNameExpressionResolver;
58+
this.settingsFilter = settingsFilter;
59+
}
60+
61+
@Override
62+
protected ClusterBlockException checkBlock(GetDataStreamSettingsAction.Request request, ClusterState state) {
63+
return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
64+
}
65+
66+
@Override
67+
protected void localClusterStateOperation(
68+
Task task,
69+
GetDataStreamSettingsAction.Request request,
70+
ClusterState state,
71+
ActionListener<GetDataStreamSettingsAction.Response> listener
72+
) throws Exception {
73+
List<String> dataStreamNames = indexNameExpressionResolver.dataStreamNames(
74+
clusterService.state(),
75+
IndicesOptions.DEFAULT,
76+
request.indices()
77+
);
78+
Map<String, DataStream> dataStreamMap = state.metadata().dataStreams();
79+
List<GetDataStreamSettingsAction.DataStreamSettingsResponse> responseList = new ArrayList<>(dataStreamNames.size());
80+
for (String dataStreamName : dataStreamNames) {
81+
DataStream dataStream = dataStreamMap.get(dataStreamName);
82+
Settings settings = settingsFilter.filter(dataStream.getSettings());
83+
Settings effectiveSettings = settingsFilter.filter(dataStream.getEffectiveSettings(state.metadata()));
84+
responseList.add(new GetDataStreamSettingsAction.DataStreamSettingsResponse(dataStreamName, settings, effectiveSettings));
85+
}
86+
listener.onResponse(new GetDataStreamSettingsAction.Response(responseList));
87+
}
88+
}

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -238,15 +238,15 @@ static GetDataStreamAction.Response innerOperation(
238238
} else {
239239
indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
240240
if (indexTemplate != null) {
241-
Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
241+
Settings settings = dataStream.getEffectiveSettings(state.metadata());
242242
ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
243243
if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
244244
indexMode = resolveMode(
245245
state,
246246
indexSettingProviders,
247247
dataStream,
248248
settings,
249-
state.metadata().templatesV2().get(indexTemplate)
249+
dataStream.getEffectiveIndexTemplate(state.metadata())
250250
);
251251
}
252252
indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);

0 commit comments

Comments
 (0)