From 79f82da651f352c16e1a5fe40c364f44c7b6611e Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 14 Jul 2020 11:45:48 +0100 Subject: [PATCH] Default to @timestamp in composable template datastream definition (#59317) This makes the data_stream timestamp field specification optional when defining a composable template. When there isn't one specified it will default to `@timestamp`. (cherry picked from commit 5609353c5d164e15a636c22019c9c17fa98aac30) Signed-off-by: Andrei Dan --- .../elasticsearch/client/IndicesClientIT.java | 2 +- ...ComposableIndexTemplatesResponseTests.java | 2 +- .../change-mappings-and-settings.asciidoc | 28 ++++----------- .../set-up-a-data-stream.asciidoc | 8 ++--- .../data-streams/use-a-data-stream.asciidoc | 11 ++++-- docs/reference/ilm/ilm-tutorial.asciidoc | 16 ++++----- .../indices/create-data-stream.asciidoc | 4 +-- .../indices/delete-data-stream.asciidoc | 4 +-- .../indices/get-data-stream.asciidoc | 4 +-- .../indices/index-templates.asciidoc | 11 +----- docs/reference/indices/resolve.asciidoc | 4 +-- .../reference/indices/rollover-index.asciidoc | 4 +-- .../10_basic_resolve_index.yml | 4 +-- .../template/ComposableTemplateIT.java | 2 +- .../metadata/ComposableIndexTemplate.java | 34 ++++++------------ .../indices/create/AutoCreateActionTests.java | 2 +- .../MetadataRolloverServiceTests.java | 6 ++-- .../ComposableIndexTemplateTests.java | 2 +- .../MetadataCreateDataStreamServiceTests.java | 4 +-- .../MetadataIndexTemplateServiceTests.java | 12 +++---- .../metadata/ToAndFromJsonMetadataTests.java | 2 +- .../authorization/alias-privileges.asciidoc | 4 +-- .../src/main/resources/logs-template.json | 4 +-- .../src/main/resources/metrics-template.json | 4 +-- .../test/data-streams/100_delete_by_query.yml | 3 +- .../test/data-streams/10_basic.yml | 15 ++++---- .../test/data-streams/110_update_by_query.yml | 3 +- .../test/data-streams/20_unsupported_apis.yml | 19 ++++------ .../30_auto_create_data_stream.yml | 3 +- .../test/data-streams/40_supported_apis.yml | 25 ++++++------- .../50_delete_backing_indices.yml | 5 ++- .../data-streams/60_get_backing_indices.yml | 3 +- .../data-streams/70_rollover_data_streams.yml | 3 +- .../80_resolve_index_data_streams.yml | 11 ++---- .../test/data-streams/90_reindex.yml | 3 +- .../datastreams/DataStreamIT.java | 8 ++--- .../test/eql/20_data_streams.yml | 3 +- .../xpack/TimeSeriesRestDriver.java | 2 +- .../ml/integration/ClassificationIT.java | 2 +- .../xpack/ml/integration/DatafeedJobsIT.java | 2 +- .../ml/integration/MlNativeIntegTestCase.java | 4 +-- .../xpack/ml/integration/RegressionIT.java | 2 +- .../10_data_stream_resolvability.yml | 36 ++++++++----------- .../test/indices.freeze/10_basic.yml | 7 ++-- .../test/security/authz/50_data_streams.yml | 3 +- .../integration/TransformRestTestCase.java | 1 - .../test/remote_cluster/10_basic.yml | 7 ++-- .../upgrades/DataStreamsUpgradeIT.java | 1 - 48 files changed, 127 insertions(+), 222 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index e4ac2d6608d35..57bc24edf9909 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -2037,7 +2037,7 @@ public void testDataStreams() throws Exception { CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"@timestamp\":{\"type\":\"date\"}}}"); Template template = new Template(null, mappings, null); ComposableIndexTemplate indexTemplate = new ComposableIndexTemplate(Collections.singletonList(dataStreamName), template, - Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate("@timestamp")); + Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate()); PutComposableIndexTemplateRequest putComposableIndexTemplateRequest = new PutComposableIndexTemplateRequest().name("ds-template").create(true).indexTemplate(indexTemplate); AcknowledgedResponse response = execute(putComposableIndexTemplateRequest, diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java index 97361fc03bd0d..23759e7b09a58 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java @@ -86,7 +86,7 @@ private static ComposableIndexTemplate randomIndexTemplate() { Long priority = randomBoolean() ? null : randomNonNegativeLong(); Long version = randomBoolean() ? null : randomNonNegativeLong(); if (randomBoolean()) { - dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate("@timestamp"); + dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate(); } return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate); } diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc index 35bea691c7e51..cfbe538dcfafe 100644 --- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc +++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc @@ -52,9 +52,7 @@ PUT /_ilm/policy/logs_policy PUT /_index_template/logs_data_stream { "index_patterns": [ "logs*" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "template": { "mappings": { "properties": { @@ -69,9 +67,7 @@ PUT /_index_template/logs_data_stream PUT /_index_template/new_logs_data_stream { "index_patterns": [ "new_logs*" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "template": { "mappings": { "properties": { @@ -124,9 +120,7 @@ for a new field, `message`, to the template. PUT /_index_template/logs_data_stream { "index_patterns": [ "logs*" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "template": { "mappings": { "properties": { @@ -193,9 +187,7 @@ mapping parameter to `true`. PUT /_index_template/logs_data_stream { "index_patterns": [ "logs*" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "template": { "mappings": { "properties": { @@ -279,9 +271,7 @@ template's `index.refresh_interval` index setting to `30s` (30 seconds). PUT /_index_template/logs_data_stream { "index_patterns": [ "logs*" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "template": { "mappings": { "properties": { @@ -345,9 +335,7 @@ The following <> requests adds new PUT /_index_template/logs_data_stream { "index_patterns": [ "logs*" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "template": { "mappings": { "properties": { @@ -460,9 +448,7 @@ uses the `logs_data_stream` template as its basis, with the following changes: PUT /_index_template/new_logs_data_stream { "index_patterns": [ "new_logs*" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "template": { "mappings": { "properties": { diff --git a/docs/reference/data-streams/set-up-a-data-stream.asciidoc b/docs/reference/data-streams/set-up-a-data-stream.asciidoc index a0013212658a4..8464af5b3c669 100644 --- a/docs/reference/data-streams/set-up-a-data-stream.asciidoc +++ b/docs/reference/data-streams/set-up-a-data-stream.asciidoc @@ -134,9 +134,7 @@ this pattern. ---- ==== -* A `data_stream` definition containing the `timestamp_field` property. - This timestamp field must be included in every document indexed to the data - stream. +* A `data_stream` object with an empty body (`{ }`). The template can also contain: @@ -176,9 +174,7 @@ configures the `logs_data_stream` template. PUT /_index_template/logs_data_stream { "index_patterns": [ "logs*" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "template": { "settings": { "index.lifecycle.name": "logs_policy" diff --git a/docs/reference/data-streams/use-a-data-stream.asciidoc b/docs/reference/data-streams/use-a-data-stream.asciidoc index 7acaeddb65f69..bc2f46931abed 100644 --- a/docs/reference/data-streams/use-a-data-stream.asciidoc +++ b/docs/reference/data-streams/use-a-data-stream.asciidoc @@ -19,8 +19,15 @@ the following: PUT /_index_template/logs_data_stream { "index_patterns": [ "logs*" ], - "data_stream": { - "timestamp_field": "@timestamp" + "data_stream": {}, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + } + } + } } } diff --git a/docs/reference/ilm/ilm-tutorial.asciidoc b/docs/reference/ilm/ilm-tutorial.asciidoc index b02337e3f843d..55de4870fd09d 100644 --- a/docs/reference/ilm/ilm-tutorial.asciidoc +++ b/docs/reference/ilm/ilm-tutorial.asciidoc @@ -121,19 +121,17 @@ with the options you specify. PUT _index_template/timeseries_template { "index_patterns": ["timeseries"], <1> - "data_stream": { - "timestamp_field": "@timestamp" <2> - }, + "data_stream": {}, "template": { "settings": { "number_of_shards": 1, "number_of_replicas": 1, - "index.lifecycle.name": "timeseries_policy" <3> + "index.lifecycle.name": "timeseries_policy" <2> }, "mappings": { "properties": { "@timestamp": { - "type": "date" <4> + "type": "date" <3> } } } @@ -143,11 +141,9 @@ PUT _index_template/timeseries_template // TEST[continued] <1> Apply the template when a document is indexed into the `timeseries` target. -<2> Identifies the timestamp field for the data source. This field must be present -in all documents indexed into the `timeseries` data stream. -<3> The name of the {ilm-init} policy used to manage the data stream. -<4> A <> or <> field mapping for the -timestamp field specified in the `timestamp_field` property +<2> The name of the {ilm-init} policy used to manage the data stream. +<3> A <> or <> field mapping for the +"@timestamp" field the data stream will use. ==== [discrete] diff --git a/docs/reference/indices/create-data-stream.asciidoc b/docs/reference/indices/create-data-stream.asciidoc index d195a514b7b85..c8f0daa71a39b 100644 --- a/docs/reference/indices/create-data-stream.asciidoc +++ b/docs/reference/indices/create-data-stream.asciidoc @@ -25,9 +25,7 @@ PUT /_index_template/template } } }, - "data_stream": { - "timestamp_field": "@timestamp" - } + "data_stream": {} } ---- //// diff --git a/docs/reference/indices/delete-data-stream.asciidoc b/docs/reference/indices/delete-data-stream.asciidoc index 62a432cfabac3..9992356bd2c57 100644 --- a/docs/reference/indices/delete-data-stream.asciidoc +++ b/docs/reference/indices/delete-data-stream.asciidoc @@ -23,9 +23,7 @@ PUT /_index_template/template } } }, - "data_stream": { - "timestamp_field": "@timestamp" - } + "data_stream": {} } PUT /_data_stream/my-data-stream diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index f041e06209e2c..b477ca6a817c0 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -35,9 +35,7 @@ PUT /_ilm/policy/my-lifecycle-policy PUT /_index_template/my-index-template { "index_patterns": [ "my-data-stream*" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "template": { "mappings": { "properties": { diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc index 052c13aa63446..72df60b518c6a 100644 --- a/docs/reference/indices/index-templates.asciidoc +++ b/docs/reference/indices/index-templates.asciidoc @@ -493,19 +493,10 @@ PUT /_index_template/template_1 } } }, - "data_stream": { - "timestamp_field": "@timestamp" - } + "data_stream": {} } -------------------------------------------------- -Required properties of a data stream definition: - -`timestamp_field`:: -(Required, string) The name of the timestamp field. This field must be present -in all documents indexed into the data stream and must be of type -<> or <>. - [[multiple-component-templates]] ===== Composing multiple component templates diff --git a/docs/reference/indices/resolve.asciidoc b/docs/reference/indices/resolve.asciidoc index 856809fc4fade..020aba8da9942 100644 --- a/docs/reference/indices/resolve.asciidoc +++ b/docs/reference/indices/resolve.asciidoc @@ -30,9 +30,7 @@ PUT /my-index/_alias/f-alias PUT /_index_template/foo_data_stream { "index_patterns": [ "foo" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "template": { "mappings": { "properties": { diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc index b9161d91fb468..1b8d124dab5ea 100644 --- a/docs/reference/indices/rollover-index.asciidoc +++ b/docs/reference/indices/rollover-index.asciidoc @@ -241,9 +241,7 @@ PUT _index_template/template } } }, - "data_stream": { - "timestamp_field": "@timestamp" - } + "data_stream": {} } ----------------------------------- // TEST diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml index 20546bcfc38d3..461bfb2e05a39 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: "7.8.99 - " - reason: "resolve index api only supported in 7.9+" + version: "7.9.99 - " + reason: "enable in 7.9+ when backported" features: allowed_warnings - do: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java index 53f9b6eeb0144..00ca9e8f6e270 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java @@ -102,7 +102,7 @@ public void testUsageOfDataStreamFails() throws IOException { Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser)); ComposableIndexTemplate template = new ComposableIndexTemplate(List.of("logs-*-*"), null, null, null, null, - null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp")); + null, new ComposableIndexTemplate.DataStreamTemplate()); Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet()); Exception actualException = (Exception) e.getCause(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 658cb41a6bc6f..ae17afae2e03f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -249,31 +250,20 @@ public String toString() { public static class DataStreamTemplate implements Writeable, ToXContentObject { - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + private static final ObjectParser PARSER = new ObjectParser<>( "data_stream_template", - args -> new DataStreamTemplate((String) args[0]) + DataStreamTemplate::new ); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), DataStream.TIMESTAMP_FIELD_FIELD); - } - - private final String timestampField; - - public DataStreamTemplate(String timestampField) { - if (FIXED_TIMESTAMP_FIELD.equals(timestampField) == false) { - throw new IllegalArgumentException("unexpected timestamp field [" + timestampField + "]"); - } - - this.timestampField = timestampField; + public DataStreamTemplate() { } public String getTimestampField() { - return timestampField; + return FIXED_TIMESTAMP_FIELD; } - DataStreamTemplate(StreamInput in) throws IOException { - this(in.readString()); + DataStreamTemplate(StreamInput in) { + this(); } /** @@ -281,18 +271,16 @@ public String getTimestampField() { */ public Map getDataSteamMappingSnippet() { return singletonMap(MapperService.SINGLE_MAPPING_NAME, singletonMap("_data_stream_timestamp", - singletonMap("path", timestampField))); + singletonMap("path", FIXED_TIMESTAMP_FIELD))); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(timestampField); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(DataStream.TIMESTAMP_FIELD_FIELD.getPreferredName(), timestampField); builder.endObject(); return builder; } @@ -300,14 +288,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public boolean equals(Object o) { if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - DataStreamTemplate that = (DataStreamTemplate) o; - return timestampField.equals(that.timestampField); + return o != null && getClass() == o.getClass(); } @Override public int hashCode() { - return Objects.hash(timestampField); + return DataStreamTemplate.class.hashCode(); } } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java index 3e7e08c1ba6b2..7cdd8561c8294 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java @@ -36,7 +36,7 @@ public void testResolveAutoCreateDataStreams() { Metadata metadata; { Metadata.Builder mdBuilder = new Metadata.Builder(); - DataStreamTemplate dataStreamTemplate = new DataStreamTemplate("@timestamp"); + DataStreamTemplate dataStreamTemplate = new DataStreamTemplate(); mdBuilder.put("1", new ComposableIndexTemplate(Collections.singletonList("legacy-logs-*"), null, null, 10L, null, null, null)); mdBuilder.put("2", new ComposableIndexTemplate(Collections.singletonList("logs-*"), null, null, 20L, null, null, dataStreamTemplate)); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 4601a4fff0c43..b40dfe9039fc6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -48,8 +48,8 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.json.JsonXContent; @@ -539,7 +539,7 @@ public void testRolloverClusterState() throws Exception { public void testRolloverClusterStateForDataStream() throws Exception { final DataStream dataStream = DataStreamTests.randomInstance(); ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"), - null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp")); + null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()); Metadata.Builder builder = Metadata.builder(); builder.put("template", template); for (Index index : dataStream.getIndices()) { @@ -638,7 +638,7 @@ public void testValidation() throws Exception { sourceIndexName = dataStream.getIndices().get(dataStream.getIndices().size() - 1).getName(); defaultRolloverIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1); ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"), - null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp")); + null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()); builder.put("template", template); for (Index index : dataStream.getIndices()) { builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java index b475faf243bbc..24efa12b3c966 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java @@ -148,7 +148,7 @@ private static ComposableIndexTemplate.DataStreamTemplate randomDataStreamTempla if (randomBoolean()) { return null; } else { - return new ComposableIndexTemplate.DataStreamTemplate("@timestamp"); + return new ComposableIndexTemplate.DataStreamTemplate(); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index 79d231e4dffce..d982f5f73985c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -46,7 +46,7 @@ public void testCreateDataStream() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = "my-data-stream"; ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStreamName + "*"), - null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp")); + null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build(); @@ -140,7 +140,7 @@ public void testCreateDataStreamNoValidTemplate() throws Exception { public static ClusterState createDataStream(final String dataStreamName) throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStreamName + "*"), - null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp")); + null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 291ec557e2815..cc5ba30dbbc03 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -847,7 +847,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex " }\n" + " }"), null), org.elasticsearch.common.collect.List.of("ct1"), 0L, 1L, null, - new ComposableIndexTemplate.DataStreamTemplate(DEFAULT_TIMESTAMP_FIELD)); + new ComposableIndexTemplate.DataStreamTemplate()); state = service.addIndexTemplateV2(state, true, "logs-data-stream-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "logs-data-stream-template", @@ -985,7 +985,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception state = service.addComponentTemplate(state, true, "ct1", ct1); ComposableIndexTemplate it = new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("logs*"), null, org.elasticsearch.common.collect.List.of("ct1"), 0L, 1L, null, - new ComposableIndexTemplate.DataStreamTemplate(DEFAULT_TIMESTAMP_FIELD)); + new ComposableIndexTemplate.DataStreamTemplate()); state = service.addIndexTemplateV2(state, true, "logs-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "logs-template", @@ -1030,7 +1030,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception " }"), null); ComposableIndexTemplate it = new ComposableIndexTemplate( org.elasticsearch.common.collect.List.of("timeseries*"), template, null, 0L, 1L, null, - new ComposableIndexTemplate.DataStreamTemplate(DEFAULT_TIMESTAMP_FIELD)); + new ComposableIndexTemplate.DataStreamTemplate()); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", @@ -1352,7 +1352,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { .build(); ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("logs-*-*"), - null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp")); + null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate()); state = service.addIndexTemplateV2(state, false, "logs", template); @@ -1396,7 +1396,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Change the pattern to one that doesn't match the data stream e = expectThrows(IllegalArgumentException.class, () -> { ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-postgres-*"), null, - null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp")); + null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate()); service.addIndexTemplateV2(stateWithDS, false, "logs", newTemplate); }); @@ -1406,7 +1406,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Add an additional template that matches our data stream at a lower priority ComposableIndexTemplate mysqlTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-mysql-*"), null, - null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp")); + null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate()); ClusterState stateWithDSAndTemplate = service.addIndexTemplateV2(stateWithDS, false, "logs-mysql", mysqlTemplate); // We should be able to replace the "logs" template, because we have the "logs-mysql" template that can handle the data stream diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index f5556f68dffef..e363350c46e08 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -78,7 +78,7 @@ public void testSimpleJsonFromAndTo() throws IOException { 5L, 4L, Collections.singletonMap("my_meta", Collections.singletonMap("potato", "chicken")), - randomBoolean() ? null : new ComposableIndexTemplate.DataStreamTemplate("@timestamp"))) + randomBoolean() ? null : new ComposableIndexTemplate.DataStreamTemplate())) .put(IndexMetadata.builder("test12") .settings(settings(Version.CURRENT) .put("setting1", "value1") diff --git a/x-pack/docs/en/security/authorization/alias-privileges.asciidoc b/x-pack/docs/en/security/authorization/alias-privileges.asciidoc index 8c1e29ebcc4dc..96969d1012b8e 100644 --- a/x-pack/docs/en/security/authorization/alias-privileges.asciidoc +++ b/x-pack/docs/en/security/authorization/alias-privileges.asciidoc @@ -43,9 +43,7 @@ backing indices, the user can retrieve a document directly from `.ds-logs-000002 PUT /_index_template/logs_data_stream { "index_patterns": [ "logs*" ], - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "template": { "mappings": { "properties": { diff --git a/x-pack/plugin/core/src/main/resources/logs-template.json b/x-pack/plugin/core/src/main/resources/logs-template.json index 84546a9c772e2..0e23872ec0170 100644 --- a/x-pack/plugin/core/src/main/resources/logs-template.json +++ b/x-pack/plugin/core/src/main/resources/logs-template.json @@ -1,9 +1,7 @@ { "index_patterns": ["logs-*-*"], "priority": 100, - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "composed_of": [ "logs-mappings", "logs-settings" diff --git a/x-pack/plugin/core/src/main/resources/metrics-template.json b/x-pack/plugin/core/src/main/resources/metrics-template.json index b13857c1c70fa..cccd852196d41 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-template.json +++ b/x-pack/plugin/core/src/main/resources/metrics-template.json @@ -1,9 +1,7 @@ { "index_patterns": ["metrics-*-*"], "priority": 100, - "data_stream": { - "timestamp_field": "@timestamp" - }, + "data_stream": {}, "composed_of": [ "metrics-mappings", "metrics-settings" diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/100_delete_by_query.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/100_delete_by_query.yml index 08a12632bea78..2e5aa17bfe233 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/100_delete_by_query.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/100_delete_by_query.yml @@ -12,8 +12,7 @@ name: my-template1 body: index_patterns: [simple-data-stream1] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/10_basic.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/10_basic.yml index 6f31510151bcc..6e36f15ed935e 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/10_basic.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/10_basic.yml @@ -3,7 +3,6 @@ setup: features: allowed_warnings version: " - 7.9.99" reason: "enable in 7.9+ when backported" - - do: allowed_warnings: - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" @@ -14,8 +13,8 @@ setup: template: settings: index.number_of_replicas: 0 - data_stream: - timestamp_field: '@timestamp' + data_stream: {} + - do: allowed_warnings: - "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" @@ -28,8 +27,7 @@ setup: properties: '@timestamp': type: date_nanos - data_stream: - timestamp_field: '@timestamp' + data_stream: {} --- "Create data stream": @@ -243,8 +241,8 @@ setup: name: generic_logs_template body: index_patterns: logs-* - data_stream: - timestamp_field: '@timestamp' + data_stream: {} + - do: index: @@ -299,8 +297,7 @@ setup: name: generic_logs_template body: index_patterns: logs-* - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: catch: bad_request diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/110_update_by_query.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/110_update_by_query.yml index 8f4477626797b..5f525e44c3e05 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/110_update_by_query.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/110_update_by_query.yml @@ -12,8 +12,7 @@ name: my-template1 body: index_patterns: [simple-data-stream1] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml index 80351c05e3897..9149eb91ebe89 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml @@ -12,8 +12,7 @@ name: my-template body: index_patterns: [logs-*] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -61,8 +60,7 @@ name: my-template1 body: index_patterns: [simple-data-stream1] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -101,7 +99,7 @@ "Prohibit shrink on data stream's write index": - skip: version: " - 7.9.99" - reason: "data streams only supported in 7.9+" + reason: "enable in 7.9+ when backported" features: allowed_warnings - do: @@ -111,8 +109,7 @@ name: my-template1 body: index_patterns: [simple-data-stream1] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -149,8 +146,7 @@ name: my-template1 body: index_patterns: [simple-data-stream1] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -171,7 +167,7 @@ "Prohibit split on data stream's write index": - skip: version: " - 7.9.99" - reason: "enable in 7.9+ when backported" + reason: "data streams only supported in 7.9+" features: allowed_warnings - do: @@ -181,8 +177,7 @@ name: my-template1 body: index_patterns: [simple-data-stream1] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/30_auto_create_data_stream.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/30_auto_create_data_stream.yml index 5b91e8fccd216..ed6e59ffe97eb 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/30_auto_create_data_stream.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/30_auto_create_data_stream.yml @@ -12,8 +12,7 @@ name: generic_logs_template body: index_patterns: logs-* - data_stream: - timestamp_field: '@timestamp' + data_stream: {} template: settings: number_of_shards: 1 diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/40_supported_apis.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/40_supported_apis.yml index ae77d23849470..3d12258ec6346 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/40_supported_apis.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/40_supported_apis.yml @@ -4,7 +4,6 @@ setup: features: allowed_warnings version: " - 7.9.99" reason: "enable in 7.9+ when backported" - - do: allowed_warnings: - "index template [logs_template] has index patterns [logs-foobar] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs_template] will take precedence during new index creation" @@ -12,8 +11,7 @@ setup: name: logs_template body: index_patterns: logs-foobar - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -41,8 +39,8 @@ teardown: --- "Verify get mapping api": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.9.99" + reason: "enable in 7.9+ when backported" - do: indices.get_mapping: @@ -53,8 +51,9 @@ teardown: --- "Verify shard stores api": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.9.99" + reason: "change to 7.8.99 after backport" + features: allowed_warnings - do: allowed_warnings: @@ -67,8 +66,7 @@ teardown: settings: number_of_shards: "1" number_of_replicas: "0" - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -115,8 +113,7 @@ teardown: name: my-template1 body: index_patterns: [simple-data-stream1] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -148,8 +145,7 @@ teardown: name: my-template1 body: index_patterns: [simple-data-stream1] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -182,8 +178,7 @@ teardown: name: my-template1 body: index_patterns: [simple-data-stream1] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/50_delete_backing_indices.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/50_delete_backing_indices.yml index ad550d66e4180..69f8b71fcbd83 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/50_delete_backing_indices.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/50_delete_backing_indices.yml @@ -10,8 +10,7 @@ setup: name: my-template body: index_patterns: [simple-*] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} --- "Delete backing index on data stream": @@ -69,7 +68,7 @@ setup: "Attempt to delete write index on data stream is rejected": - skip: version: " - 7.9.99" - reason: "enable in 7.9+ when backported" + reason: "enable in 7.9+ when backported9+" - do: indices.create_data_stream: diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/60_get_backing_indices.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/60_get_backing_indices.yml index 23f1e26392b13..785b7632cadd7 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/60_get_backing_indices.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/60_get_backing_indices.yml @@ -12,8 +12,7 @@ name: my-template body: index_patterns: [data-*] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/70_rollover_data_streams.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/70_rollover_data_streams.yml index f7056c52face1..08ffac907ac4e 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/70_rollover_data_streams.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/70_rollover_data_streams.yml @@ -12,8 +12,7 @@ name: my-template body: index_patterns: [data-*] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/80_resolve_index_data_streams.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/80_resolve_index_data_streams.yml index f8516488f2068..478bda2471aa8 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/80_resolve_index_data_streams.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/80_resolve_index_data_streams.yml @@ -12,8 +12,7 @@ setup: name: my-template1 body: index_patterns: [simple-data-stream1] - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: allowed_warnings: @@ -22,13 +21,7 @@ setup: name: my-template2 body: index_patterns: [simple-data-stream2] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/90_reindex.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/90_reindex.yml index 28c1c68d8984b..204ecb62a243c 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/90_reindex.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/90_reindex.yml @@ -12,8 +12,7 @@ setup: name: generic_logs_template body: index_patterns: logs-* - data_stream: - timestamp_field: '@timestamp' + data_stream: {} --- teardown: diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 4224cf4cd46ba..9a0ba9bd745fe 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -294,7 +294,7 @@ public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Except null, null, null, - new ComposableIndexTemplate.DataStreamTemplate("@timestamp") + new ComposableIndexTemplate.DataStreamTemplate() ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); @@ -376,7 +376,7 @@ public void testTimeStampValidationInvalidFieldMapping() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate("@timestamp") + new ComposableIndexTemplate.DataStreamTemplate() ) ); @@ -837,7 +837,7 @@ public void testMixedAutoCreate() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate("@timestamp") + new ComposableIndexTemplate.DataStreamTemplate() ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet(); @@ -1005,7 +1005,7 @@ static void putComposableIndexTemplate( null, null, null, - new ComposableIndexTemplate.DataStreamTemplate("@timestamp") + new ComposableIndexTemplate.DataStreamTemplate() ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); diff --git a/x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/20_data_streams.yml b/x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/20_data_streams.yml index 0e4e01c197775..c7f889c241501 100644 --- a/x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/20_data_streams.yml +++ b/x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/20_data_streams.yml @@ -17,8 +17,7 @@ properties: '@timestamp': type: date - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java index 9f633d6e5c06d..617524fa48368 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java @@ -125,7 +125,7 @@ public static void createComposableTemplate(RestClient client, String templateNa StringEntity templateJSON = new StringEntity( String.format(Locale.ROOT, "{\n" + " \"index_patterns\": \"%s\",\n" + - " \"data_stream\": { \"timestamp_field\": \"@timestamp\" },\n" + + " \"data_stream\": {},\n" + " \"template\": %s\n" + "}", indexPattern, Strings.toString(builder)), ContentType.APPLICATION_JSON); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java index dcc8ad89488ef..392ab2cb7a5c5 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java @@ -729,7 +729,7 @@ private static void createIndex(String index, boolean isDatastream) { " }"; if (isDatastream) { try { - createDataStreamAndTemplate(index, "@timestamp", mapping); + createDataStreamAndTemplate(index, mapping); } catch (IOException ex) { throw new ElasticsearchException(ex); } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java index 03525e76c3ded..8780347b52207 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java @@ -114,7 +114,7 @@ public void testLookbackOnlyDataStream() throws Exception { " }" + " }\n" + " }"; - createDataStreamAndTemplate("datafeed_data_stream", "@timestamp", mapping); + createDataStreamAndTemplate("datafeed_data_stream", mapping); long numDocs = randomIntBetween(32, 2048); long now = System.currentTimeMillis(); long oneWeekAgo = now - 604800000; diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index bec9d15a85799..975c983cd3bde 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -275,7 +275,7 @@ protected void ensureClusterStateConsistency() throws IOException { } } - protected static void createDataStreamAndTemplate(String dataStreamName, String timeField, String mapping) throws IOException { + protected static void createDataStreamAndTemplate(String dataStreamName, String mapping) throws IOException { client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request(dataStreamName + "_template") .indexTemplate(new ComposableIndexTemplate(Collections.singletonList(dataStreamName), @@ -284,7 +284,7 @@ protected static void createDataStreamAndTemplate(String dataStreamName, String null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(timeField)))) + new ComposableIndexTemplate.DataStreamTemplate()))) .actionGet(); client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet(); } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java index a0851918d7bb3..7453bf176a656 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java @@ -487,7 +487,7 @@ static void indexData(String sourceIndex, int numTrainingRows, int numNonTrainin " }"; if (dataStream) { try { - createDataStreamAndTemplate(sourceIndex, "@timestamp", mapping); + createDataStreamAndTemplate(sourceIndex, mapping); } catch (IOException ex) { throw new ElasticsearchException(ex); } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml index bb9fffd7865c9..33990977ecfa9 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml @@ -1,8 +1,8 @@ --- "Verify data stream resolvability for xpack apis": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.9.99" + reason: "enable in 7.9+ when backported" features: allowed_warnings - do: @@ -17,8 +17,7 @@ properties: '@timestamp': type: date - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -54,8 +53,8 @@ --- "Verify data stream resolvability in ilm explain api": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.9.99" + reason: "enable in 7.9+ when backported" features: allowed_warnings - do: @@ -88,8 +87,7 @@ name: generic_logs_template body: index_patterns: logs-* - data_stream: - timestamp_field: '@timestamp' + data_stream: {} template: settings: index.lifecycle.name: "my_lifecycle" @@ -119,8 +117,8 @@ --- "Verify data stream resolvability rollup": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.9.99" + reason: "enable in 7.9+ when backported" features: allowed_warnings - do: @@ -135,8 +133,7 @@ properties: '@timestamp': type: date - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -196,8 +193,7 @@ type: date settings: index.lifecycle.name: "my_moveable_timeseries_lifecycle" - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -261,8 +257,7 @@ properties: '@timestamp': type: date - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -318,8 +313,7 @@ properties: '@timestamp': type: date - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -358,8 +352,7 @@ properties: '@timestamp': type: date - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: @@ -433,8 +426,7 @@ type: keyword price: type: integer - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/10_basic.yml index 051dee0328d95..c49e958e80a26 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/10_basic.yml @@ -136,8 +136,8 @@ --- "Cannot freeze write index for data stream": - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" + version: " - 7.9.99" + reason: "enable in 7.9+ when backported" features: allowed_warnings - do: @@ -152,8 +152,7 @@ properties: '@timestamp': type: date - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/50_data_streams.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/50_data_streams.yml index e10890191d11d..ebeb19baf172c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/50_data_streams.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/50_data_streams.yml @@ -61,8 +61,7 @@ setup: properties: '@timestamp': type: date - data_stream: - timestamp_field: '@timestamp' + data_stream: {} --- teardown: diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java index 3cc84c06ff883..f7b7107e4a3cd 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java @@ -121,7 +121,6 @@ protected void createReviewsIndex(String indexName, int numDocs, String dateType "{\n" + " \"index_patterns\": [ \"" + indexName + "\" ],\n" + " \"data_stream\": {\n" + - " \"timestamp_field\": \"@timestamp\"\n" + " },\n" + " \"template\": \n" + Strings.toString(builder) + "}" diff --git a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml index 01cb27150846c..a87eb9bd63bb8 100644 --- a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml +++ b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml @@ -68,8 +68,8 @@ setup: properties: '@timestamp': type: date - data_stream: - timestamp_field: '@timestamp' + data_stream: {} + - do: allowed_warnings: - "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" @@ -82,8 +82,7 @@ setup: properties: '@timestamp': type: date - data_stream: - timestamp_field: '@timestamp' + data_stream: {} - do: indices.create_data_stream: diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java index 5a85e2e41c936..cafc42e61d821 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java @@ -33,7 +33,6 @@ public void testDataStreams() throws IOException { " }\n" + " },\n" + " \"data_stream\":{\n" + - " \"timestamp_field\":\"@timestamp\"" + " }\n" + " }"; Request request = new Request("PUT", "/_index_template/1");