From 0b0cf4c175a4032621b7f2e0431e6513eef4b100 Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 15 Feb 2022 14:03:24 +0800 Subject: [PATCH 1/9] fix tsdb/05_dimension_and_metric_in_non_tsdb_index/nested dimensions --- .../test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml index 04b2df3359068..78c933ab6e6f2 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml @@ -218,8 +218,8 @@ no _tsid in standard indices: --- nested dimensions: - skip: - version: all - reason: Awaits fix https://github.com/elastic/elasticsearch/issues/83915 + version: " - 8.0.99" + reason: introduced in 8.1.0 - do: indices.create: @@ -244,5 +244,3 @@ nested dimensions: "@timestamp": "2021-04-28T18:35:24.467Z" nested: - dim: foo - - dim: bar - - dim: baz From b826168d3e26760592908548d50955df1d02bf07 Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 15 Feb 2022 16:28:23 +0800 Subject: [PATCH 2/9] add test --- ...dimension_and_metric_in_non_tsdb_index.yml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml index 78c933ab6e6f2..e7165ba1fe005 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml @@ -244,3 +244,36 @@ nested dimensions: "@timestamp": "2021-04-28T18:35:24.467Z" nested: - dim: foo + +--- +muiti fields nested dimensions: + - skip: + version: " - 8.0.99" + reason: introduced in 8.1.0 + + - do: + indices.create: + index: test + body: + mappings: + properties: + "@timestamp": + type: date + nested: + type: nested + properties: + dim: + type: keyword + time_series_dimension: true + + - do: + catch: /Dimension field \[nested.dim\] cannot be a multi-valued field./ + index: + index: test + refresh: true + body: + "@timestamp": "2021-04-28T18:35:24.467Z" + nested: + - dim: foo + - dim: bar + - dim: baz From 78b373bbb456fa233f77bd95c6ac89ee535425d9 Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 15 Feb 2022 23:59:58 +0800 Subject: [PATCH 3/9] changelog --- docs/changelog/83920.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/83920.yaml diff --git a/docs/changelog/83920.yaml b/docs/changelog/83920.yaml new file mode 100644 index 0000000000000..69c833e840bcf --- /dev/null +++ b/docs/changelog/83920.yaml @@ -0,0 +1,5 @@ +pr: 83920 +summary: "TSDB: fix tsdb/05_dimension_and_metric_in_non_tsdb_index/nested dimensions" +area: TSDB +type: bug +issues: [] From 743a14dc4c8979bcc599f5568901afdbe7b9a332 Mon Sep 17 00:00:00 2001 From: weizijun Date: Wed, 16 Feb 2022 15:09:24 +0800 Subject: [PATCH 4/9] no nested dimensions --- ...dimension_and_metric_in_non_tsdb_index.yml | 49 ++--------------- .../index/mapper/IpFieldMapper.java | 9 ++++ .../index/mapper/KeywordFieldMapper.java | 9 ++++ .../index/mapper/NumberFieldMapper.java | 9 ++++ .../index/mapper/NestedObjectMapperTests.java | 52 +++++++++++++++++++ .../unsignedlong/UnsignedLongFieldMapper.java | 9 ++++ 6 files changed, 92 insertions(+), 45 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml index e7165ba1fe005..4b6a376637617 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml @@ -216,42 +216,13 @@ no _tsid in standard indices: - is_false: fields._tsid # _tsid metadata field must not exist in non-time-series indices --- -nested dimensions: +no nested dimensions: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 - - - do: - indices.create: - index: test - body: - mappings: - properties: - "@timestamp": - type: date - nested: - type: nested - properties: - dim: - type: keyword - time_series_dimension: true - - - do: - index: - index: test - refresh: true - body: - "@timestamp": "2021-04-28T18:35:24.467Z" - nested: - - dim: foo - ---- -muiti fields nested dimensions: - - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.1.99" + reason: introduced in 8.2.0 - do: + catch: /time_series_dimension can't be configured in nested field \[nested.dim\]/ indices.create: index: test body: @@ -265,15 +236,3 @@ muiti fields nested dimensions: dim: type: keyword time_series_dimension: true - - - do: - catch: /Dimension field \[nested.dim\] cannot be a multi-valued field./ - index: - index: test - refresh: true - body: - "@timestamp": "2021-04-28T18:35:24.467Z" - nested: - - dim: foo - - dim: bar - - dim: baz diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 901f0bda6801f..ab1b580d7a5be 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -520,4 +520,13 @@ protected void indexScriptValues( public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(dimension).init(this); } + + @Override + public void doValidate(MappingLookup lookup) { + if (dimension && null != lookup.nestedLookup().getNestedParent(name())) { + throw new IllegalArgumentException( + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + " can't be configured in nested field [" + name() + "]" + ); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index dd934733d4a3c..aebba68868c81 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -955,4 +955,13 @@ protected String contentType() { public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), indexAnalyzers, scriptCompiler).dimension(dimension).init(this); } + + @Override + public void doValidate(MappingLookup lookup) { + if (dimension && null != lookup.nestedLookup().getNestedParent(name())) { + throw new IllegalArgumentException( + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + " can't be configured in nested field [" + name() + "]" + ); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 6abfd25f194b0..bc6e9fb1f1ef6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1501,4 +1501,13 @@ public FieldMapper.Builder getMergeBuilder() { .metric(metricType) .init(this); } + + @Override + public void doValidate(MappingLookup lookup) { + if (dimension && null != lookup.nestedLookup().getNestedParent(name())) { + throw new IllegalArgumentException( + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + " can't be configured in nested field [" + name() + "]" + ); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 5355ecb676323..0794521939a4c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.ObjectMapper.Dynamic; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; @@ -23,9 +24,12 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Arrays; import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.function.Function; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -1380,4 +1384,52 @@ public void testFieldNamesIncludeInRoot() throws Exception { assertThat(doc.docs().get(4).get("_field_names"), nullValue()); } + public void testNoDimensionNestedFields() { + { + Exception e = expectThrows(IllegalArgumentException.class, () -> createDocumentMapper(mapping(b -> { + b.startObject("nested"); + { + b.field("type", "nested"); + b.startObject("properties"); + { + b.startObject("foo") + .field("type", randomFrom(List.of("keyword", "ip", "long", "double", "unsigned_long"))) + .field("time_series_dimension", true) + .endObject(); + } + b.endObject(); + } + b.endObject(); + }))); + assertThat(e.getMessage(), containsString("time_series_dimension can't be configured in nested field [nested.foo]")); + } + + { + Exception e = expectThrows(IllegalArgumentException.class, () -> createDocumentMapper(mapping(b -> { + b.startObject("nested"); + { + b.field("type", "nested"); + b.startObject("properties"); + { + b.startObject("other").field("type", "keyword").endObject(); + b.startObject("object").field("type", "object"); + { + b.startObject("properties"); + { + b.startObject("foo") + .field("type", randomFrom(List.of("keyword", "ip", "long", "double", "unsigned_long"))) + .field("time_series_dimension", true) + .endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + }))); + assertThat(e.getMessage(), containsString("time_series_dimension can't be configured in nested field [nested.object.foo]")); + } + } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 2300644128733..153e3f63dc71a 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.SimpleMappedFieldType; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; @@ -648,4 +649,12 @@ protected static long sortableSignedLongToUnsigned(long value) { return value ^ MASK_2_63; } + @Override + public void doValidate(MappingLookup lookup) { + if (dimension && null != lookup.nestedLookup().getNestedParent(name())) { + throw new IllegalArgumentException( + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + " can't be configured in nested field [" + name() + "]" + ); + } + } } From bec9e12cdb3596ce3dd14fd0f964fe515008433a Mon Sep 17 00:00:00 2001 From: weizijun Date: Wed, 16 Feb 2022 15:44:14 +0800 Subject: [PATCH 5/9] changelog --- docs/changelog/83920.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog/83920.yaml b/docs/changelog/83920.yaml index 69c833e840bcf..95bfc2536fc84 100644 --- a/docs/changelog/83920.yaml +++ b/docs/changelog/83920.yaml @@ -1,5 +1,5 @@ pr: 83920 -summary: "TSDB: fix tsdb/05_dimension_and_metric_in_non_tsdb_index/nested dimensions" +summary: "TSDB: Reject the nested object fields that are configured time_series_dimension" area: TSDB -type: bug +type: enhancement issues: [] From feeb6e84d92e6491b444e9e53e854fdb6d951fe0 Mon Sep 17 00:00:00 2001 From: weizijun Date: Wed, 16 Feb 2022 23:12:34 +0800 Subject: [PATCH 6/9] checkstyle --- .../elasticsearch/index/mapper/NestedObjectMapperTests.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 0794521939a4c..4d6505d620a6e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MapperService.MergeReason; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.ObjectMapper.Dynamic; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; @@ -24,12 +23,10 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.function.Function; -import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; From f4b06e3642f85696b85b0c598d1e6d12aece970d Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 17 Feb 2022 00:18:07 +0800 Subject: [PATCH 7/9] fixup --- .../rest-api-spec/test/tsdb/20_mapping.yml | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index 3774526653b03..59f480e42c98a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -312,7 +312,7 @@ nested dimensions: reason: message changed in 8.2.0 - do: - catch: /cannot have nested fields when index is in \[index.mode=time_series\]/ + catch: /time_series_dimension can't be configured in nested field \[nested.dim\]/ indices.create: index: test body: @@ -333,3 +333,34 @@ nested dimensions: dim: type: keyword time_series_dimension: true + +--- +nested fields: + - skip: + version: " - 8.1.99" + reason: message changed in 8.2.0 + + - do: + catch: /cannot have nested fields when index is in \[index.mode=time_series\]/ + indices.create: + index: test + body: + settings: + index: + mode: time_series + routing_path: [dim] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + dim: + type: keyword + time_series_dimension: true + nested: + type: nested + properties: + foo: + type: keyword From e49ad14295fa46c57a4a9468b1cdfe7c0e4d7f95 Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 17 Feb 2022 00:58:12 +0800 Subject: [PATCH 8/9] fixup --- .../org/elasticsearch/index/mapper/NestedObjectMapperTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 4d6505d620a6e..bfde3f8ec064c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -1414,7 +1414,7 @@ public void testNoDimensionNestedFields() { b.startObject("properties"); { b.startObject("foo") - .field("type", randomFrom(List.of("keyword", "ip", "long", "double", "unsigned_long"))) + .field("type", randomFrom(List.of("keyword", "ip", "long", "short", "integer", "byte", "unsigned_long"))) .field("time_series_dimension", true) .endObject(); } From c9cf46dd47eecccf853a32965bf2128885f61d90 Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 17 Feb 2022 01:06:24 +0800 Subject: [PATCH 9/9] fixup --- .../elasticsearch/index/mapper/NestedObjectMapperTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index bfde3f8ec064c..2d5d12dd599b0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -1390,7 +1390,7 @@ public void testNoDimensionNestedFields() { b.startObject("properties"); { b.startObject("foo") - .field("type", randomFrom(List.of("keyword", "ip", "long", "double", "unsigned_long"))) + .field("type", randomFrom(List.of("keyword", "ip", "long", "short", "integer", "byte"))) .field("time_series_dimension", true) .endObject(); } @@ -1414,7 +1414,7 @@ public void testNoDimensionNestedFields() { b.startObject("properties"); { b.startObject("foo") - .field("type", randomFrom(List.of("keyword", "ip", "long", "short", "integer", "byte", "unsigned_long"))) + .field("type", randomFrom(List.of("keyword", "ip", "long", "short", "integer", "byte"))) .field("time_series_dimension", true) .endObject(); }