diff --git a/docs/changelog/105073.yaml b/docs/changelog/105073.yaml new file mode 100644 index 0000000000000..62b22f17cf4ea --- /dev/null +++ b/docs/changelog/105073.yaml @@ -0,0 +1,5 @@ +pr: 105073 +summary: Support non-keyword fields as dimension subfields of pass-through object +area: TSDB +type: enhancement +issues: [] diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index 08b09bbc78348..772cc0f98d757 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -260,7 +260,7 @@ public void testInvalidTsdbTemplatesNoTimeSeriesDimensionAttribute() throws Exce assertThat( e.getCause().getCause().getMessage(), equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [metricset] was not a dimension." ) @@ -289,7 +289,7 @@ public void testInvalidTsdbTemplatesNoTimeSeriesDimensionAttribute() throws Exce } } - public void testInvalidTsdbTemplatesNoKeywordFieldType() throws Exception { + public void testTsdbTemplatesNoKeywordFieldType() throws Exception { var mappingTemplate = """ { "_doc":{ @@ -315,18 +315,7 @@ public void testInvalidTsdbTemplatesNoKeywordFieldType() throws Exception { .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build() ); - Exception e = expectThrows( - IllegalArgumentException.class, - () -> client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet() - ); - assertThat( - e.getCause().getCause().getMessage(), - equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [metricset] was [long]." - ) - ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); } public void testInvalidTsdbTemplatesMissingSettings() throws Exception { diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index 5d84baa5f6ea4..0c05da5fe539a 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -53,6 +53,17 @@ public class TSDBPassthroughIndexingIT extends ESSingleNodeTestCase { public static final String MAPPING_TEMPLATE = """ { "_doc":{ + "dynamic_templates": [ + { + "strings_as_ip": { + "match_mapping_type": "string", + "match": "*ip", + "mapping": { + "type": "ip" + } + } + } + ], "properties": { "@timestamp" : { "type": "date" @@ -87,6 +98,8 @@ public class TSDBPassthroughIndexingIT extends ESSingleNodeTestCase { "@timestamp": "$time", "attributes": { "metricset": "pod", + "number.long": $number1, + "number.double": $number2, "pod": { "name": "$name", "uid": "$uid", @@ -102,6 +115,15 @@ public class TSDBPassthroughIndexingIT extends ESSingleNodeTestCase { } """; + private static String getRandomDoc(Instant time) { + return DOC.replace("$time", formatInstant(time)) + .replace("$uid", randomUUID()) + .replace("$name", randomAlphaOfLength(4)) + .replace("$number1", Long.toString(randomLong())) + .replace("$number2", Double.toString(randomDouble())) + .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean()))); + } + @Override protected Collection> getPlugins() { return List.of(DataStreamsPlugin.class, InternalSettingsPlugin.class); @@ -137,13 +159,7 @@ public void testIndexingGettingAndSearching() throws Exception { Instant time = Instant.now(); for (int i = 0; i < indexingIters; i++) { var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); - indexRequest.source( - DOC.replace("$time", formatInstant(time)) - .replace("$uid", randomUUID()) - .replace("$name", randomAlphaOfLength(4)) - .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean()))), - XContentType.JSON - ); + indexRequest.source(getRandomDoc(time), XContentType.JSON); var indexResponse = client().index(indexRequest).actionGet(); index = indexResponse.getIndex(); String id = indexResponse.getId(); @@ -176,7 +192,9 @@ public void testIndexingGettingAndSearching() throws Exception { ); @SuppressWarnings("unchecked") var attributes = (Map>) ObjectPath.eval("properties.attributes.properties", mapping); - assertMap(attributes.get("pod.ip"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true)); + assertMap(attributes.get("number.long"), matchesMap().entry("type", "long").entry("time_series_dimension", true)); + assertMap(attributes.get("number.double"), matchesMap().entry("type", "float").entry("time_series_dimension", true)); + assertMap(attributes.get("pod.ip"), matchesMap().entry("type", "ip").entry("time_series_dimension", true)); assertMap(attributes.get("pod.uid"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true)); assertMap(attributes.get("pod.name"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true)); // alias field mappers: @@ -184,6 +202,14 @@ public void testIndexingGettingAndSearching() throws Exception { ObjectPath.eval("properties.metricset", mapping), matchesMap().entry("type", "alias").entry("path", "attributes.metricset") ); + assertMap( + ObjectPath.eval("properties.number.properties.long", mapping), + matchesMap().entry("type", "alias").entry("path", "attributes.number.long") + ); + assertMap( + ObjectPath.eval("properties.number.properties.double", mapping), + matchesMap().entry("type", "alias").entry("path", "attributes.number.double") + ); assertMap( ObjectPath.eval("properties.pod.properties", mapping), matchesMap().extraOk().entry("name", matchesMap().entry("type", "alias").entry("path", "attributes.pod.name")) @@ -220,13 +246,7 @@ public void testIndexingGettingAndSearchingShrunkIndex() throws Exception { var bulkRequest = new BulkRequest(dataStreamName); for (int i = 0; i < numBulkItems; i++) { var indexRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE); - indexRequest.source( - DOC.replace("$time", formatInstant(time)) - .replace("$uid", randomUUID()) - .replace("$name", randomAlphaOfLength(4)) - .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean()))), - XContentType.JSON - ); + indexRequest.source(getRandomDoc(time), XContentType.JSON); bulkRequest.add(indexRequest); time = time.plusMillis(1); } @@ -276,7 +296,8 @@ public void testIndexingGettingAndSearchingShrunkIndex() throws Exception { searchRequest.source(new SearchSourceBuilder().query(new TermQueryBuilder("_id", id))); assertResponse(client().search(searchRequest), searchResponse -> { assertHitCount(searchResponse, 1); - assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo(id)); + var getResponse2 = client().get(new GetRequest(shrunkenTarget, searchResponse.getHits().getHits()[0].getId())).actionGet(); + assertThat(getResponse2.isExists(), is(true)); }); } } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 278c14c09a31a..d54a5b6dd1179 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -230,13 +230,13 @@ dynamic templates: refresh: true body: - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:08.138Z", "data": "10", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10" }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z", "data": "10", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:09.138Z", "data": "20", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10" }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z", "data": "20", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.138Z", "data": "30", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20" }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z", "data": "30", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.238Z", "data": "40", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20" }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z", "data": "40", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5" }' - do: search: @@ -262,7 +262,7 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiK8yYWLhfZ18WLDvTuBX1YJX1Ll7UMNJqYNES5Eg" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -281,7 +281,7 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiK8yYWLhfZ18WLDvTuBX1YJX1Ll7UMNJqYNES5Eg" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -300,7 +300,7 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiK8yYWLhfZ18WLDvTuBX1YJX1Ll7UMNJqYNES5Eg" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -312,14 +312,14 @@ dynamic templates: filterA: filter: term: - another.dim2: 10 + another.dim2: 10.5 aggs: tsids: terms: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiK8yYWLhfZ18WLDvTuBX1YJX1Ll7UMNJqYNES5Eg" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -463,13 +463,13 @@ dynamic templates with nesting: refresh: true body: - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10" }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10" }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20" }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20" }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5" }' - do: search: @@ -495,7 +495,7 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK6pWmkqIGKYiosO9O4X2dfFL8p_4TfsFAUUYYv9EqSmEQ" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -514,7 +514,7 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK6pWmkqIGKYiosO9O4X2dfFL8p_4TfsFAUUYYv9EqSmEQ" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -533,7 +533,7 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK6pWmkqIGKYiosO9O4X2dfFL8p_4TfsFAUUYYv9EqSmEQ" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -545,14 +545,14 @@ dynamic templates with nesting: filterA: filter: term: - another.dim2: 10 + another.dim2: 10.5 aggs: tsids: terms: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK6pWmkqIGKYiosO9O4X2dfFL8p_4TfsFAUUYYv9EqSmEQ" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml index 83f8aab34e02a..7d323085241d1 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml @@ -43,15 +43,15 @@ location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } + - match: { _id: AAAAAAAAAYhw9-f4KBNikT0PtwhSVNrMwdFLUCCMxM33PPpaqsD0ZiHcNlFdRDOaemCHLb4 } - match: { result: created } - match: { _version: 1 } - do: delete: index: weather_sensors - id: crxuhAep5Npwt_etAAABiHD35_g - - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } + id: AAAAAAAAAYhw9-f4KBNikT0PtwhSVNrMwdFLUCCMxM33PPpaqsD0ZiHcNlFdRDOaemCHLb4 + - match: { _id: AAAAAAAAAYhw9-f4KBNikT0PtwhSVNrMwdFLUCCMxM33PPpaqsD0ZiHcNlFdRDOaemCHLb4 } - match: { result: deleted } - match: { _version: 2 } @@ -68,6 +68,6 @@ location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } + - match: { _id: AAAAAAAAAYhw9-f4KBNikT0PtwhSVNrMwdFLUCCMxM33PPpaqsD0ZiHcNlFdRDOaemCHLb4 } - match: { result: created } - match: { _version: 3 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 5813445326ef6..53a1821ee8351 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -90,11 +90,11 @@ missing routing path field: --- missing dimension on routing path field: - skip: - version: " - 8.7.99" - reason: error message changed in 8.8.0 + version: " - 8.12.99" + reason: error message changed in 8.13.0 - do: - catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[tag\] was not a dimension./' + catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[tag\] was not a dimension./' indices.create: index: test body: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index 1ff32192b9e08..e2f464860e6ff 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -121,11 +121,11 @@ top level wildcard dim object: --- exact match object type: - skip: - version: " - 8.7.99" - reason: routing_path error message updated in 8.8.0 + version: " - 8.12.99" + reason: routing_path error message updated in 8.13.0 - do: - catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[dim\] was \[object\]./' + catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[dim\] was \[object\]./' indices.create: index: tsdb_index body: @@ -158,7 +158,7 @@ non keyword matches routing_path: reason: routing_path error message updated in 8.8.0 - do: - catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[@timestamp\] was \[date\]./' + catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[@timestamp\] was \[date\]./' indices.create: index: test_index body: @@ -273,7 +273,7 @@ runtime field matching routing path: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim_kw": "dim", "dim": {"foo": "a"}, "extra_field": 100}' - - match: {items.0.index.error.reason: "All fields that match routing_path must be keywords with [time_series_dimension: true] or flattened fields with a list of dimensions in [time_series_dimensions] and without the [script] parameter. [dim.foo] was a runtime [keyword]."} + - match: {items.0.index.error.reason: "All fields that match routing_path must be configured with [time_series_dimension: true] or flattened fields with a list of dimensions in [time_series_dimensions] and without the [script] parameter. [dim.foo] was a runtime [keyword]."} --- "dynamic: false matches routing_path": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 6ef03ba8ebcc4..53a4e6ed407ec 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -76,7 +76,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} + - match: {items.0.index._id: AAIAAAAAAXkZ0u3zKCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o} - do: bulk: @@ -85,7 +85,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} + - match: {items.0.index._id: AAIAAAAAAXkZ0u3zKCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o} - do: search: @@ -97,39 +97,39 @@ generates a consistent id: - match: {hits.total.value: 9} - - match: { hits.hits.0._id: cn4excfoxSs_KdA5AAABeRnRFAY } + - match: { hits.hits.0._id: AAIAAAAAAXkZ0RQGKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ } - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z } - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.1._id: cZZNs7B9sSWsyrL5AAABeRnRGTM } + - match: { hits.hits.1._id: AAIAAAAAAXkZ0RkzKCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o } - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:04.467Z } - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.2._id: cn4excfoxSs_KdA5AAABeRnRYiY } + - match: { hits.hits.2._id: AAIAAAAAAXkZ0WImKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ } - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:23.142Z } - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.3._id: cZZNs7B9sSWsyrL5AAABeRnRZ1M } + - match: { hits.hits.3._id: AAIAAAAAAXkZ0WdTKCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o } - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:24.467Z } - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.4._id: cZZNs7B9sSWsyrL5AAABeRnRtXM } + - match: { hits.hits.4._id: AAIAAAAAAXkZ0bVzKCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o } - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:44.467Z } - match: { hits.hits.4._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.5._id: cn4excfoxSs_KdA5AAABeRnR11Y } + - match: { hits.hits.5._id: AAIAAAAAAXkZ0ddWKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ } - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:50:53.142Z } - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.6._id: cn4excfoxSs_KdA5AAABeRnR_mY } + - match: { hits.hits.6._id: AAIAAAAAAXkZ0f5mKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ } - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:03.142Z } - match: { hits.hits.6._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.7._id: cZZNs7B9sSWsyrL5AAABeRnSA5M } + - match: { hits.hits.7._id: AAIAAAAAAXkZ0gOTKCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o } - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:51:04.467Z } - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.8._id: cZZNs7B9sSWsyrL5AAABeRnS7fM } + - match: { hits.hits.8._id: AAIAAAAAAXkZ0u3zKCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o } - match: { hits.hits.8._source.@timestamp: 2021-04-28T18:52:04.467Z } - match: { hits.hits.8._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } @@ -170,7 +170,7 @@ index a new document on top of an old one: network: tx: 111434595272 rx: 430605511 - - match: {_id: cn4excfoxSs_KdA5AAABeRnR_mY} + - match: {_id: AAIAAAAAAXkZ0f5mKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ} - do: search: @@ -215,7 +215,7 @@ index a new document on top of an old one over bulk: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: {items.0.index._id: cn4excfoxSs_KdA5AAABeRnR_mY} + - match: {items.0.index._id: AAIAAAAAAXkZ0f5mKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ} - do: search: @@ -239,7 +239,7 @@ create operation on top of old document fails: reason: id generation changed in 8.2 - do: - catch: "/\\[cn4excfoxSs_KdA5AAABeRnR_mY\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" + catch: "/\\[AAIAAAAAAXkZ0f5mKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" index: refresh: true index: test @@ -268,7 +268,7 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[cn4excfoxSs_KdA5AAABeRnR_mY][KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[AAIAAAAAAXkZ0f5mKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ][KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: @@ -284,12 +284,12 @@ ids query: - field: k8s.pod.network.tx query: ids: - values: ["cn4excfoxSs_KdA5AAABeRnR11Y", "cn4excfoxSs_KdA5AAABeRnR_mY"] + values: ["AAIAAAAAAXkZ0ddWKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ", "AAIAAAAAAXkZ0f5mKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"] sort: ["@timestamp"] - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "cn4excfoxSs_KdA5AAABeRnR11Y"} + - match: {hits.hits.0._id: "AAIAAAAAAXkZ0ddWKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434587694]} - - match: {hits.hits.1._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } + - match: {hits.hits.1._id: "AAIAAAAAAXkZ0f5mKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [1434595272]} --- @@ -301,9 +301,9 @@ get: - do: get: index: test - id: cZZNs7B9sSWsyrL5AAABeRnSA5M + id: AAIAAAAAAXkZ0gOTKCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o - match: {_index: test} - - match: {_id: cZZNs7B9sSWsyrL5AAABeRnSA5M} + - match: {_id: AAIAAAAAAXkZ0gOTKCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o} - match: _source: "@timestamp": "2021-04-28T18:51:04.467Z" @@ -351,7 +351,7 @@ delete: - do: delete: index: test - id: cn4excfoxSs_KdA5AAABeRnR_mY + id: AAIAAAAAAXkZ0f5mKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ - match: {result: deleted} --- @@ -391,20 +391,20 @@ delete over _bulk: mget: index: test body: - ids: [ cn4excfoxSs_KdA5AAABeRnR_mY, cn4excfoxSs_KdA5AAABeRnR11Y ] + ids: [ AAIAAAAAAXkZ0f5mKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ, AAIAAAAAAXkZ0ddWKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ ] - match: { docs.0._index: "test" } - - match: { docs.0._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } + - match: { docs.0._id: "AAIAAAAAAXkZ0f5mKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } - match: { docs.0.found: true } - match: { docs.1._index: "test" } - - match: { docs.1._id: "cn4excfoxSs_KdA5AAABeRnR11Y" } + - match: { docs.1._id: "AAIAAAAAAXkZ0ddWKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } - match: { docs.1.found: true } - do: bulk: index: test body: - - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR_mY"}}' - - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR11Y"}}' + - '{"delete": {"_id": "AAIAAAAAAXkZ0f5mKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}}' + - '{"delete": {"_id": "AAIAAAAAAXkZ0ddWKCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}}' - '{"delete": {"_id": "not found ++ not found"}}' - match: {items.0.delete.result: deleted} - match: {items.1.delete.result: deleted} @@ -454,7 +454,7 @@ routing_path matches deep object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"bar": {"baz": {"uid": "uid1"}}}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: OcEOGchJrjH1fFX8AAABeRnRGTM} + - match: {items.0.index._id: AAAAAAAAAXkZ0RkzJM470g16EHdYU4GgBL87HDcWrvVxXB-xBpIYlWsMx6RFR8fHaQ} --- routing_path matches object: @@ -495,4 +495,4 @@ routing_path matches object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"uid": "uid1"}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: 8bgiqW9JKwAyp1bZAAABeRnRGTM} + - match: {items.0.index._id: AAAAAAAAAXkZ0RkzJFvylKRG_nnAwbD-SKLruAAWrvVxXB-xBpIYlWsMx6RFR8fHaQ} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 22672756bdaf0..1bae1df4650a1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -2848,14 +2848,14 @@ public static Set selectShrinkShards(int shardId, IndexMetadata sourceI } /** - * Returns the routing factor for and shrunk index with the given number of target shards. + * Returns the routing factor for a shrunk index with the given number of target shards. * This factor is used in the hash function in * {@link IndexRouting#indexShard} to guarantee consistent * hashing / routing of documents even if the number of shards changed (ie. a shrunk index). * * @param sourceNumberOfShards the total number of shards in the source index * @param targetNumberOfShards the total number of shards in the target index - * @return the routing factor for and shrunk index with the given number of target shards. + * @return the routing factor for a shrunk index with the given number of target shards. * @throws IllegalArgumentException if the number of source shards is less than the number of target shards or if the source shards * are not divisible by the number of target shards. */ diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 1ed9d759c4ca8..73412c2778c01 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -109,13 +109,31 @@ private IndexRouting(IndexMetadata metadata) { public abstract void collectSearchShards(String routing, IntConsumer consumer); /** - * Convert a hash generated from an {@code (id, routing}) pair into a - * shard id. + * Convert a hash generated from an {@code (id, routing}) pair into a shard id. + * This needs to be updated in tandem with calculateRoutingHash below, see also + * {@link IndexMetadata#getRoutingFactor(int, int)}. */ protected final int hashToShardId(int hash) { return Math.floorMod(hash, routingNumShards) / routingFactor; } + /** + * Returns a routing hash that can be used to route a request to the same shard. + */ + public static int calculateRoutingHash(int shardId, int routingFactor) { + /* This is the inverse of the function above. It also works in case of shrinking, e.g. + * - Say an index is shrunk from 8 to 2 shards. routingNumShards stays the same, routingFactor value is 4x. + * - Shard with shardID 5 gets mapped to shardId 1 after shrinking. + * - The calculation becomes: + * new shardId = (((old shardId) * (old routingFactor)) % routingNumShards) / (new routingFactor) + * = (old shardId) / 4 = 5 / 4 = 1 + * + * Note that the ids generated after shrinking will include the updated values for shardId and routingFactor, + * so they'll get mapped to the right shard as usual. + */ + return shardId * routingFactor; + } + /** * Convert a routing value into a hash. */ @@ -248,10 +266,6 @@ public static class ExtractFromSource extends IndexRouting { this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true); } - public boolean matchesField(String fieldName) { - return isRoutingPath.test(fieldName); - } - @Override public void process(IndexRequest indexRequest) {} @@ -334,16 +348,13 @@ private void extractItem(String path, XContentParser source) throws IOException source.nextToken(); break; case VALUE_STRING: + case VALUE_NUMBER: hashes.add(new NameAndHash(new BytesRef(path), hash(new BytesRef(source.text())))); source.nextToken(); break; case VALUE_NULL: source.nextToken(); break; - case VALUE_NUMBER: // allow parsing numbers assuming routing fields are always keyword fields - hashes.add(new NameAndHash(new BytesRef(path), hash(new BytesRef(source.text())))); - source.nextToken(); - break; default: throw new ParsingException( source.getTokenLocation(), diff --git a/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java b/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java index 52ddaa9a87589..a4e450eda9319 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java @@ -9,6 +9,7 @@ package org.elasticsearch.common.lucene.uid; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; @@ -16,13 +17,18 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo; import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion; +import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; @@ -117,10 +123,11 @@ final class PerThreadIDVersionAndSeqNoLookup { * using the same cache key. Otherwise we'd have to disable caching * entirely for these readers. */ - public DocIdAndVersion lookupVersion(BytesRef id, boolean loadSeqNo, LeafReaderContext context) throws IOException { + public DocIdAndVersion lookupVersion(BytesRef id, boolean loadSeqNo, LeafReaderContext context, boolean searchFallback) + throws IOException { assert readerKey == null || context.reader().getCoreCacheHelper().getKey().equals(readerKey) : "context's reader is not the same as the reader class was initialized on."; - int docID = getDocID(id, context); + int docID = getDocID(id, context, searchFallback); if (docID != DocIdSetIterator.NO_MORE_DOCS) { final long seqNo; @@ -143,23 +150,36 @@ public DocIdAndVersion lookupVersion(BytesRef id, boolean loadSeqNo, LeafReaderC * returns the internal lucene doc id for the given id bytes. * {@link DocIdSetIterator#NO_MORE_DOCS} is returned if not found * */ - private int getDocID(BytesRef id, LeafReaderContext context) throws IOException { + private int getDocID(BytesRef id, LeafReaderContext context, boolean searchFallback) throws IOException { // termsEnum can possibly be null here if this leaf contains only no-ops. - if (termsEnum != null && termsEnum.seekExact(id)) { - final Bits liveDocs = context.reader().getLiveDocs(); - int docID = DocIdSetIterator.NO_MORE_DOCS; - // there may be more than one matching docID, in the case of nested docs, so we want the last one: - docsEnum = termsEnum.postings(docsEnum, 0); - for (int d = docsEnum.nextDoc(); d != DocIdSetIterator.NO_MORE_DOCS; d = docsEnum.nextDoc()) { - if (liveDocs != null && liveDocs.get(d) == false) { - continue; + if (termsEnum != null) { + if (termsEnum.seekExact(id)) { + final Bits liveDocs = context.reader().getLiveDocs(); + int docID = DocIdSetIterator.NO_MORE_DOCS; + // there may be more than one matching docID, in the case of nested docs, so we want the last one: + docsEnum = termsEnum.postings(docsEnum, 0); + for (int d = docsEnum.nextDoc(); d != DocIdSetIterator.NO_MORE_DOCS; d = docsEnum.nextDoc()) { + if (liveDocs != null && liveDocs.get(d) == false) { + continue; + } + docID = d; } - docID = d; + return docID; + } + if (searchFallback && id.length > 20) { + // There's no inverted index available, decode the id and perform a lookup using the timestamp and the tsid. + long timestamp = ByteUtils.readLongBE(id.bytes, 4); + BytesRef tsid = new BytesRef(id.bytes, 12, id.length - 12); + + IndexSearcher searcher = new IndexSearcher(context.reader()); + BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder(); + booleanQuery.add(SortedSetDocValuesField.newSlowExactQuery("_tsid", tsid), BooleanClause.Occur.FILTER); + booleanQuery.add(LongPoint.newRangeQuery("@timestamp", timestamp, timestamp), BooleanClause.Occur.FILTER); + TopDocs topDocs = searcher.search(booleanQuery.build(), 1); + return (topDocs.scoreDocs.length > 0) ? topDocs.scoreDocs[0].doc : DocIdSetIterator.NO_MORE_DOCS; } - return docID; - } else { - return DocIdSetIterator.NO_MORE_DOCS; } + return DocIdSetIterator.NO_MORE_DOCS; } private static long readNumericDocValues(LeafReader reader, String field, int docId) throws IOException { @@ -175,7 +195,7 @@ private static long readNumericDocValues(LeafReader reader, String field, int do DocIdAndSeqNo lookupSeqNo(BytesRef id, LeafReaderContext context) throws IOException { assert readerKey == null || context.reader().getCoreCacheHelper().getKey().equals(readerKey) : "context's reader is not the same as the reader class was initialized on."; - final int docID = getDocID(id, context); + final int docID = getDocID(id, context, false); if (docID != DocIdSetIterator.NO_MORE_DOCS) { final long seqNo = readNumericDocValues(context.reader(), SeqNoFieldMapper.NAME, docID); return new DocIdAndSeqNo(docID, seqNo, context); diff --git a/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java b/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java index 56c0869992bac..0de1446566010 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java @@ -136,7 +136,8 @@ public static class DocIdAndSeqNo { *
  • a doc ID and a version otherwise * */ - public static DocIdAndVersion timeSeriesLoadDocIdAndVersion(IndexReader reader, Term term, boolean loadSeqNo) throws IOException { + public static DocIdAndVersion timeSeriesLoadDocIdAndVersion(IndexReader reader, Term term, boolean loadSeqNo, boolean searchFallback) + throws IOException { PerThreadIDVersionAndSeqNoLookup[] lookups = getLookupState(reader, term.field(), false); List leaves = reader.leaves(); // iterate backwards to optimize for the frequently updated documents @@ -144,7 +145,7 @@ public static DocIdAndVersion timeSeriesLoadDocIdAndVersion(IndexReader reader, for (int i = leaves.size() - 1; i >= 0; i--) { final LeafReaderContext leaf = leaves.get(i); PerThreadIDVersionAndSeqNoLookup lookup = lookups[leaf.ord]; - DocIdAndVersion result = lookup.lookupVersion(term.bytes(), loadSeqNo, leaf); + DocIdAndVersion result = lookup.lookupVersion(term.bytes(), loadSeqNo, leaf, searchFallback); if (result != null) { return result; } @@ -171,10 +172,8 @@ public static DocIdAndVersion timeSeriesLoadDocIdAndVersion(IndexReader reader, public static DocIdAndVersion timeSeriesLoadDocIdAndVersion(IndexReader reader, Term uid, String id, boolean loadSeqNo) throws IOException { byte[] idAsBytes = Base64.getUrlDecoder().decode(id); - assert idAsBytes.length == 20; - // id format: [4 bytes (basic hash routing fields), 8 bytes prefix of 128 murmurhash dimension fields, 8 bytes - // @timestamp) - long timestamp = ByteUtils.readLongBE(idAsBytes, 12); + // id format: [4 bytes (basic hash routing fields), 8 bytes @timestamp, 20 bytes _tsid) + long timestamp = ByteUtils.readLongBE(idAsBytes, 4); PerThreadIDVersionAndSeqNoLookup[] lookups = getLookupState(reader, uid.field(), true); List leaves = reader.leaves(); @@ -190,7 +189,7 @@ public static DocIdAndVersion timeSeriesLoadDocIdAndVersion(IndexReader reader, if (timestamp > lookup.maxTimestamp) { return null; } - DocIdAndVersion result = lookup.lookupVersion(uid.bytes(), loadSeqNo, leaf); + DocIdAndVersion result = lookup.lookupVersion(uid.bytes(), loadSeqNo, leaf, false); if (result != null) { return result; } @@ -204,7 +203,7 @@ public static DocIdAndVersion loadDocIdAndVersionUncached(IndexReader reader, Te for (int i = leaves.size() - 1; i >= 0; i--) { final LeafReaderContext leaf = leaves.get(i); PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(leaf.reader(), term.field(), false, false); - DocIdAndVersion result = lookup.lookupVersion(term.bytes(), loadSeqNo, leaf); + DocIdAndVersion result = lookup.lookupVersion(term.bytes(), loadSeqNo, leaf, false); if (result != null) { return result; } diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index ae6185cdcc6b6..4d89cdb66bf27 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -101,6 +101,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion NEW_INDEXVERSION_FORMAT = def(8_501_00_0, Version.LUCENE_9_9_1); public static final IndexVersion UPGRADE_LUCENE_9_9_2 = def(8_502_00_0, Version.LUCENE_9_9_2); public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_502_00_1, Version.LUCENE_9_9_2); + public static final IndexVersion TIME_SERIES_SHARD_ID_IN_ID = def(8_503_00_0, Version.LUCENE_9_9_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index a910e496ce1b5..a81afe9a6bfba 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -653,7 +653,7 @@ protected final GetResult getFromSearcher(Get get, Engine.Searcher searcher, boo if (uncachedLookup) { docIdAndVersion = VersionsAndSeqNoResolver.loadDocIdAndVersionUncached(searcher.getIndexReader(), get.uid(), true); } else { - docIdAndVersion = VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(searcher.getIndexReader(), get.uid(), true); + docIdAndVersion = VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(searcher.getIndexReader(), get.uid(), true, true); } } catch (Exception e) { Releasables.closeWhileHandlingException(searcher); diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 0b2e83532d030..e432a1a3cb888 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -1021,7 +1021,8 @@ private VersionValue resolveDocVersion(final Operation op, boolean loadSeqNo) th docIdAndVersion = VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion( searcher.getIndexReader(), op.uid(), - loadSeqNo + loadSeqNo, + false ); } } diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java index 708d042c91bf9..956459119b712 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java @@ -259,7 +259,8 @@ private LeafReader createInMemoryLeafReader() { Map.of(), DocumentSizeObserver.EMPTY_INSTANCE ), - mappingLookup + mappingLookup, + shardId.id() ); parsedDocs.updateSeqID(operation.seqNo(), operation.primaryTerm()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java index e9bf5838be8b3..cf453bd1571be 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java @@ -208,7 +208,7 @@ protected final LeafFactory leafFactory(SearchExecutionContext context) { public void validateMatchedRoutingPath(final String routingPath) { throw new IllegalArgumentException( "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " + + "must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] " + "and without the [script] parameter. [" + name() diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index f7dc09cdbb370..c5b26850b9753 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -89,7 +89,11 @@ public MappingLookup mappers() { } public ParsedDocument parse(SourceToParse source) throws DocumentParsingException { - return documentParser.parseDocument(source, mappingLookup); + return parse(source, 0); + } + + public ParsedDocument parse(SourceToParse source, int shardId) throws DocumentParsingException { + return documentParser.parseDocument(source, mappingLookup, shardId); } public void validate(IndexSettings settings, boolean checkLimits) { @@ -125,7 +129,7 @@ public void validate(IndexSettings settings, boolean checkLimits) { // object type is not allowed in the routing paths if (path.equals(objectName)) { throw new IllegalArgumentException( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] " + "and without the [script] parameter. [" + objectName diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 9a0e391102708..ebf97ba7161e9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -67,7 +67,7 @@ public final class DocumentParser { * @return the parsed document * @throws DocumentParsingException whenever there's a problem parsing the document */ - public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingLookup) throws DocumentParsingException { + public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingLookup, int shardId) throws DocumentParsingException { if (source.source() != null && source.source().length() == 0) { throw new DocumentParsingException(new XContentLocation(0, 0), "failed to parse, document is empty"); } @@ -80,7 +80,7 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL XContentHelper.createParser(parserConfiguration, source.source(), xContentType) ) ) { - context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser); + context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser, shardId); validateStart(context.parser()); MetadataFieldMapper[] metadataFieldsMappers = mappingLookup.getMapping().getSortedMetadataMappers(); internalParseDocument(metadataFieldsMappers, context); @@ -839,14 +839,16 @@ private static class RootDocumentParserContext extends DocumentParserContext { MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse source, - XContentParser parser + XContentParser parser, + int shardId ) throws IOException { super( mappingLookup, mappingParserContext, source, mappingLookup.getMapping().getRoot(), - ObjectMapper.Dynamic.getRootDynamic(mappingLookup) + ObjectMapper.Dynamic.getRootDynamic(mappingLookup), + shardId ); if (mappingLookup.getMapping().getRoot().subobjects()) { this.parser = DotExpandingXContentParser.expandDots(parser, this.path); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 01e67377adafd..ee50990849b28 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -115,6 +115,8 @@ public int get() { private final Set fieldsAppliedFromTemplates; private final Set copyToFields; + private final int shardId; + private DocumentParserContext( MappingLookup mappingLookup, MappingParserContext mappingParserContext, @@ -131,7 +133,8 @@ private DocumentParserContext( ObjectMapper.Dynamic dynamic, Set fieldsAppliedFromTemplates, Set copyToFields, - DynamicMapperSize dynamicMapperSize + DynamicMapperSize dynamicMapperSize, + final int shardId ) { this.mappingLookup = mappingLookup; this.mappingParserContext = mappingParserContext; @@ -149,6 +152,7 @@ private DocumentParserContext( this.fieldsAppliedFromTemplates = fieldsAppliedFromTemplates; this.copyToFields = copyToFields; this.dynamicMappersSize = dynamicMapperSize; + this.shardId = shardId; } private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, DocumentParserContext in) { @@ -168,7 +172,8 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, dynamic, in.fieldsAppliedFromTemplates, in.copyToFields, - in.dynamicMappersSize + in.dynamicMappersSize, + in.shardId ); } @@ -177,7 +182,8 @@ protected DocumentParserContext( MappingParserContext mappingParserContext, SourceToParse source, ObjectMapper parent, - ObjectMapper.Dynamic dynamic + ObjectMapper.Dynamic dynamic, + int shardId ) { this( mappingLookup, @@ -195,7 +201,8 @@ protected DocumentParserContext( dynamic, new HashSet<>(), new HashSet<>(), - new DynamicMapperSize() + new DynamicMapperSize(), + shardId ); } @@ -231,6 +238,10 @@ public final SourceToParse sourceToParse() { return this.sourceToParse; } + public int getShardId() { + return shardId; + } + /** * Add the given {@code field} to the set of ignored fields. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 71fd9edd49903..c1423499b8c75 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1415,6 +1415,19 @@ private static boolean isDeprecatedParameter(String propName, IndexVersion index } } + public abstract static class DimensionBuilder extends Builder { + + protected boolean isDimension = false; + + public DimensionBuilder(String name) { + super(name); + } + + void setDimension() { + this.isDimension = true; + } + } + public static BiConsumer notInMultiFields(String type) { return (n, c) -> { if (c.isWithinMultiField()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index c965a77f1b5bf..f178cb1bca34e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -36,8 +36,13 @@ static IdLoader fromLeafStoredFieldLoader() { /** * @return returns an {@link IdLoader} instance that syn synthesizes _id from routing, _tsid and @timestamp fields. */ - static IdLoader createTsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths) { - return new TsIdLoader(indexRouting, routingPaths); + static IdLoader createTsIdLoader( + IndexRouting.ExtractFromSource indexRouting, + List routingPaths, + int shardId, + int routingFactor + ) { + return new TsIdLoader(indexRouting, routingPaths, shardId, routingFactor); } Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException; @@ -59,28 +64,35 @@ final class TsIdLoader implements IdLoader { private final IndexRouting.ExtractFromSource indexRouting; private final List routingPaths; + private final int shardId; + private final int routingFactor; - TsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths) { + TsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths, int shardId, int routingFactor) { this.routingPaths = routingPaths; this.indexRouting = indexRouting; + this.shardId = shardId; + this.routingFactor = routingFactor; } public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException { - IndexRouting.ExtractFromSource.Builder[] builders = new IndexRouting.ExtractFromSource.Builder[docIdsInLeaf.length]; - for (int i = 0; i < builders.length; i++) { - builders[i] = indexRouting.builder(); - } + IndexRouting.ExtractFromSource.Builder[] builders = null; + if (indexRouting != null) { + builders = new IndexRouting.ExtractFromSource.Builder[docIdsInLeaf.length]; + for (int i = 0; i < builders.length; i++) { + builders[i] = indexRouting.builder(); + } - for (String routingField : routingPaths) { - // Routing field must always be keyword fields, so it is ok to use SortedSetDocValues directly here. - SortedSetDocValues dv = DocValues.getSortedSet(reader, routingField); - for (int i = 0; i < docIdsInLeaf.length; i++) { - int docId = docIdsInLeaf[i]; - var builder = builders[i]; - if (dv.advanceExact(docId)) { - for (int j = 0; j < dv.docValueCount(); j++) { - BytesRef routingValue = dv.lookupOrd(dv.nextOrd()); - builder.addMatching(routingField, routingValue); + for (String routingField : routingPaths) { + // Routing field must always be keyword fields, so it is ok to use SortedSetDocValues directly here. + SortedSetDocValues dv = DocValues.getSortedSet(reader, routingField); + for (int i = 0; i < docIdsInLeaf.length; i++) { + int docId = docIdsInLeaf[i]; + var builder = builders[i]; + if (dv.advanceExact(docId)) { + for (int j = 0; j < dv.docValueCount(); j++) { + BytesRef routingValue = dv.lookupOrd(dv.nextOrd()); + builder.addMatching(routingField, routingValue); + } } } } @@ -100,9 +112,12 @@ public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] assert found; assert timestampDocValues.docValueCount() == 1; long timestamp = timestampDocValues.nextValue(); - - var routingBuilder = builders[i]; - ids[i] = TsidExtractingIdFieldMapper.createId(false, routingBuilder, tsid, timestamp, new byte[16]); + if (builders != null) { + var routingBuilder = builders[i]; + ids[i] = TsidExtractingIdFieldMapper.createId(false, routingBuilder, tsid, timestamp, new byte[16]); + } else { + ids[i] = TsidExtractingIdFieldMapper.createId(shardId, routingFactor, tsid, timestamp); + } } return new TsIdLeaf(docIdsInLeaf, ids); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 355b38d4dcb96..d9051e69e34f7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -69,7 +69,7 @@ private static IpFieldMapper toType(FieldMapper in) { return (IpFieldMapper) in; } - public static final class Builder extends FieldMapper.Builder { + public static final class Builder extends FieldMapper.DimensionBuilder { private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -166,7 +166,7 @@ protected Parameter[] getParameters() { @Override public IpFieldMapper build(MapperBuilderContext context) { - if (context.parentObjectContainsDimensions()) { + if (super.isDimension || context.parentObjectContainsDimensions()) { dimension.setValue(true); } return new IpFieldMapper( @@ -246,6 +246,21 @@ public boolean mayExistInIndex(SearchExecutionContext context) { return context.fieldExistsInIndex(name()); } + @Override + public boolean isDimension() { + return isDimension; + } + + @Override + public boolean supportsDimension() { + return true; + } + + @Override + public boolean hasScriptValues() { + return scriptValues != null; + } + private static InetAddress parse(Object value) { if (value instanceof InetAddress) { return (InetAddress) value; @@ -461,13 +476,6 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi } return terms.intersect(prefixAutomaton, searchBytes); } - - /** - * @return true if field has been marked as a dimension field - */ - public boolean isDimension() { - return isDimension; - } } private final boolean indexed; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 06e689784b087..e33f8b593d80d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -137,7 +137,7 @@ private static KeywordFieldMapper toType(FieldMapper in) { return (KeywordFieldMapper) in; } - public static final class Builder extends FieldMapper.Builder { + public static final class Builder extends FieldMapper.DimensionBuilder { private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -304,7 +304,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType } else if (splitQueriesOnWhitespace.getValue()) { searchAnalyzer = Lucene.WHITESPACE_ANALYZER; } - if (context.parentObjectContainsDimensions()) { + if (super.isDimension || context.parentObjectContainsDimensions()) { dimension(true); } return new KeywordFieldType( @@ -811,35 +811,19 @@ public int ignoreAbove() { return ignoreAbove; } - /** - * @return true if field has been marked as a dimension field - */ @Override public boolean isDimension() { return isDimension; } @Override - public void validateMatchedRoutingPath(final String routingPath) { - if (false == isDimension) { - throw new IllegalArgumentException( - "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] was not a dimension." - ); - } - if (scriptValues != null) { - throw new IllegalArgumentException( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] has a [script] parameter." - ); - } + public boolean supportsDimension() { + return true; + } + + @Override + public boolean hasScriptValues() { + return scriptValues != null; } public boolean hasNormalizer() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 265374a687312..c3cbf47c25c1a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -195,6 +195,20 @@ public boolean isDimension() { return false; } + /** + * @return true if field can be marked as a dimension field + */ + public boolean supportsDimension() { + return false; + } + + /** + * @return true if field has script values. + */ + public boolean hasScriptValues() { + return false; + } + /** * @return a list of dimension fields. Expected to be used by fields that have * nested fields or that, in some way, identify a collection of fields by means @@ -621,16 +635,39 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi * Validate that this field can be the target of {@link IndexMetadata#INDEX_ROUTING_PATH}. */ public void validateMatchedRoutingPath(String routingPath) { - throw new IllegalArgumentException( - "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] was [" - + typeName() - + "]." - ); + if (supportsDimension() == false) { + throw new IllegalArgumentException( + "All fields that match routing_path " + + "must be configured with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + name() + + "] was [" + + typeName() + + "]." + ); + } + + if (hasScriptValues()) { + throw new IllegalArgumentException( + "All fields that match routing_path must be configured with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + name() + + "] has a [script] parameter." + ); + } + + if (isDimension() == false) { + throw new IllegalArgumentException( + "All fields that match routing_path " + + "must be configured with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + name() + + "] was not a dimension." + ); + } } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 2245e527c2aa2..79419548a95c2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -70,7 +70,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -89,7 +88,7 @@ private static NumberFieldMapper toType(FieldMapper in) { private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099); - public static final class Builder extends FieldMapper.Builder { + public static final class Builder extends FieldMapper.DimensionBuilder { private final Parameter indexed; private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -183,11 +182,6 @@ public Builder( } }); this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> { - if (v && EnumSet.of(NumberType.INTEGER, NumberType.LONG, NumberType.BYTE, NumberType.SHORT).contains(type) == false) { - throw new IllegalArgumentException( - "Parameter [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] cannot be set to numeric type [" + type.name + "]" - ); - } if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { throw new IllegalArgumentException( "Field [" @@ -267,7 +261,7 @@ protected Parameter[] getParameters() { @Override public NumberFieldMapper build(MapperBuilderContext context) { - if (context.parentObjectContainsDimensions()) { + if (super.isDimension || context.parentObjectContainsDimensions()) { dimension.setValue(true); } @@ -1745,13 +1739,21 @@ public CollapseType collapseType() { return CollapseType.NUMERIC; } - /** - * @return true if field has been marked as a dimension field - */ + @Override public boolean isDimension() { return isDimension; } + @Override + public boolean supportsDimension() { + return true; + } + + @Override + public boolean hasScriptValues() { + return scriptValues != null; + } + /** * If field is a time series metric field, returns its metric type * @return the metric type or null diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java index 05ae7e59f69c3..ed22b0f53a1e7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -41,8 +41,8 @@ public Builder(String name) { @Override public PassThroughObjectMapper.Builder add(Mapper.Builder builder) { - if (timeSeriesDimensionSubFields.value() && builder instanceof KeywordFieldMapper.Builder keywordBuilder) { - keywordBuilder.dimension(true); + if (timeSeriesDimensionSubFields.value() && builder instanceof FieldMapper.DimensionBuilder dimensionBuilder) { + dimensionBuilder.setDimension(); } super.add(builder); return this; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 1ee7caff497ad..94a44b1c97072 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -140,7 +140,11 @@ public void postParse(DocumentParserContext context) throws IOException { ? timeSeriesIdBuilder.buildLegacyTsid().toBytesRef() : timeSeriesIdBuilder.buildTsidHash().toBytesRef(); context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); - TsidExtractingIdFieldMapper.createField(context, timeSeriesIdBuilder.routingBuilder, timeSeriesId); + TsidExtractingIdFieldMapper.createField( + context, + getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_SHARD_ID_IN_ID) ? timeSeriesIdBuilder.routingBuilder : null, + timeSeriesId + ); } private IndexVersion getIndexVersionCreated(final DocumentParserContext context) { @@ -407,6 +411,8 @@ public static Map decodeTsidAsMap(StreamInput in) { Object ul = DocValueFormat.UNSIGNED_LONG_SHIFTED.format(in.readLong()); result.put(name, ul); } + case (byte) 'd' -> // parse a double + result.put(name, in.readDouble()); default -> throw new IllegalArgumentException("Cannot parse [" + name + "]: Unknown type [" + type + "]"); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 1e613767c2c89..522af5ef559d7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; +import java.util.Base64; import java.util.Locale; /** @@ -52,22 +53,26 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra ); } long timestamp = timestampField.numericValue().longValue(); - byte[] suffix = new byte[16]; - String id = createId(context.hasDynamicMappers(), routingBuilder, tsid, timestamp, suffix); - /* - * Make sure that _id from extracting the tsid matches that _id - * from extracting the _source. This should be true for all valid - * documents with valid mappings. *But* some invalid mappings - * will not parse the field but be rejected later by the dynamic - * mappings machinery. So if there are any dynamic mappings - * at all we just skip the assertion because we can't be sure - * it always must pass. - */ - IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); - assert context.getDynamicMappers().isEmpty() == false - || context.getDynamicRuntimeFields().isEmpty() == false - || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); - + String id; + if (routingBuilder != null) { + byte[] suffix = new byte[16]; + id = createId(context.hasDynamicMappers(), routingBuilder, tsid, timestamp, suffix); + /* + * Make sure that _id from extracting the tsid matches that _id + * from extracting the _source. This should be true for all valid + * documents with valid mappings. *But* some invalid mappings + * will not parse the field but be rejected later by the dynamic + * mappings machinery. So if there are any dynamic mappings + * at all we just skip the assertion because we can't be sure + * it always must pass. + */ + IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); + assert context.getDynamicMappers().isEmpty() == false + || context.getDynamicRuntimeFields().isEmpty() == false + || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); + } else { + id = createId(context.getShardId(), context.indexSettings().getIndexMetadata().getRoutingFactor(), tsid, timestamp); + } if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { throw new IllegalArgumentException( String.format( @@ -85,6 +90,14 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra context.doc().add(new StringField(NAME, uidEncoded, Field.Store.YES)); } + public static String createId(int shardId, int routingFactor, BytesRef tsid, long timestamp) { + byte[] bytes = new byte[12 + tsid.length]; + ByteUtils.writeIntLE(IndexRouting.calculateRoutingHash(shardId, routingFactor), bytes, 0); + ByteUtils.writeLongBE(timestamp, bytes, 4); // Big Ending shrinks the inverted index by ~37% + System.arraycopy(tsid.bytes, 0, bytes, 12, tsid.length); + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + public static String createId( boolean dynamicMappersExists, IndexRouting.ExtractFromSource.Builder routingBuilder, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 5a8efb6c8ed59..c502371db6dea 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -231,6 +231,16 @@ public static final class KeyedFlattenedFieldType extends StringFieldType { private final String rootName; private final boolean isDimension; + @Override + public boolean isDimension() { + return isDimension; + } + + @Override + public boolean supportsDimension() { + return true; + } + KeyedFlattenedFieldType( String rootName, boolean indexed, @@ -280,24 +290,6 @@ public Query existsQuery(SearchExecutionContext context) { return new PrefixQuery(term); } - @Override - public void validateMatchedRoutingPath(final String routingPath) { - if (false == isDimension) { - throw new IllegalArgumentException( - "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + this.rootName - + "." - + this.key - + "] was [" - + typeName() - + "]." - ); - } - } - @Override public Query rangeQuery( Object lowerTerm, @@ -730,6 +722,11 @@ public boolean isDimension() { return isDimension; } + @Override + public boolean supportsDimension() { + return true; + } + @Override public List dimensions() { return this.dimensions; @@ -737,17 +734,8 @@ public List dimensions() { @Override public void validateMatchedRoutingPath(final String routingPath) { - if (false == isDimension && this.dimensions.contains(routingPath) == false) { - throw new IllegalArgumentException( - "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] was [" - + typeName() - + "]." - ); + if (this.dimensions.contains(routingPath) == false) { + super.validateMatchedRoutingPath(routingPath); } } } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 86af6d21b7a09..d0dfcd5367bf7 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -359,7 +359,7 @@ public boolean hasNamedQueries() { * Parse a document with current mapping. */ public ParsedDocument parseDocument(SourceToParse source) throws DocumentParsingException { - return mapperService.documentParser().parseDocument(source, mappingLookup); + return mapperService.documentParser().parseDocument(source, mappingLookup, shardId); } public NestedLookup nestedLookup() { diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 3bafe139756fd..730c7be091166 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -980,7 +980,8 @@ private Engine.IndexResult applyIndexOperation( isRetry, ifSeqNo, ifPrimaryTerm, - getRelativeTimeInNanos() + getRelativeTimeInNanos(), + shardId().id() ); Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); if (update != null) { @@ -1017,7 +1018,8 @@ public static Engine.Index prepareIndex( boolean isRetry, long ifSeqNo, long ifPrimaryTerm, - long startTimeInNanos + long startTimeInNanos, + int shardId ) { assert source.dynamicTemplates().isEmpty() || origin == Engine.Operation.Origin.PRIMARY : "dynamic_templates parameter can only be associated with primary operations"; @@ -1027,7 +1029,7 @@ public static Engine.Index prepareIndex( documentMapper = DocumentMapper.createEmpty(mapperService); mapping = documentMapper.mapping(); } - ParsedDocument doc = documentMapper.parse(source); + ParsedDocument doc = documentMapper.parse(source, shardId); if (mapping != null) { // If we are indexing but there is no mapping we create one. This is to ensure that whenever at least a document is indexed // some mappings do exist. It covers for the case of indexing an empty doc (`{}`). diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index b4c0b200eb143..dcc4b796af9b1 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -315,7 +315,7 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect ); DocumentParser documentParser = indexShard.mapperService().documentParser(); MappingLookup mappingLookup = indexShard.mapperService().mappingLookup(); - ParsedDocument parsedDocument = documentParser.parseDocument(source, mappingLookup); + ParsedDocument parsedDocument = documentParser.parseDocument(source, mappingLookup, indexShard.shardId().id()); // select the right fields and generate term vectors LuceneDocument doc = parsedDocument.rootDoc(); Set seenFields = new HashSet<>(); diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 7f9e808db9560..e1f997ff043fb 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -27,15 +27,14 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.mapper.IdLoader; -import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.query.AbstractQueryBuilder; @@ -76,8 +75,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; import java.util.function.LongSupplier; @@ -897,23 +894,18 @@ public SourceLoader newSourceLoader() { @Override public IdLoader newIdLoader() { if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { - var indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting(); - List routingPaths = indexService.getMetadata().getRoutingPaths(); - for (String routingField : routingPaths) { - if (routingField.contains("*")) { - // In case the routing fields include path matches, find any matches and add them as distinct fields - // to the routing path. - Set matchingRoutingPaths = new TreeSet<>(routingPaths); - for (Mapper mapper : indexService.mapperService().mappingLookup().fieldMappers()) { - if (mapper instanceof KeywordFieldMapper && indexRouting.matchesField(mapper.name())) { - matchingRoutingPaths.add(mapper.name()); - } - } - routingPaths = new ArrayList<>(matchingRoutingPaths); - break; - } + IndexRouting.ExtractFromSource indexRouting = null; + List routingPaths = null; + if (indexService.getIndexSettings().getIndexVersionCreated().before(IndexVersions.TIME_SERIES_SHARD_ID_IN_ID)) { + indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting(); + routingPaths = indexService.getMetadata().getRoutingPaths(); } - return IdLoader.createTsIdLoader(indexRouting, routingPaths); + return IdLoader.createTsIdLoader( + indexRouting, + routingPaths, + request.shardId().id(), + indexService.getMetadata().getRoutingFactor() + ); } else { return IdLoader.fromLeafStoredFieldLoader(); } diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java index b463a5ddf11a9..679c04161c683 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java @@ -57,19 +57,19 @@ public void testSimple() throws Exception { LeafReaderContext segment = reader.leaves().get(0); PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME, false); // found doc - DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment); + DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment, false); assertNotNull(result); assertEquals(87, result.version); assertEquals(0, result.docId); // not found doc - assertNull(lookup.lookupVersion(new BytesRef("7"), randomBoolean(), segment)); + assertNull(lookup.lookupVersion(new BytesRef("7"), randomBoolean(), segment, false)); // deleted doc writer.deleteDocuments(new Term(IdFieldMapper.NAME, "6")); reader.close(); reader = DirectoryReader.open(writer); segment = reader.leaves().get(0); lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME, false); - assertNull(lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment)); + assertNull(lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment, false)); reader.close(); writer.close(); dir.close(); @@ -93,7 +93,7 @@ public void testTwoDocuments() throws Exception { LeafReaderContext segment = reader.leaves().get(0); PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME, false); // return the last doc when there are duplicates - DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment); + DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment, false); assertNotNull(result); assertEquals(87, result.version); assertEquals(1, result.docId); @@ -103,7 +103,7 @@ public void testTwoDocuments() throws Exception { reader = DirectoryReader.open(writer); segment = reader.leaves().get(0); lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME, false); - result = lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment); + result = lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment, false); assertNotNull(result); assertEquals(87, result.version); assertEquals(1, result.docId); @@ -113,7 +113,7 @@ public void testTwoDocuments() throws Exception { reader = DirectoryReader.open(writer); segment = reader.leaves().get(0); lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME, false); - assertNull(lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment)); + assertNull(lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment, false)); reader.close(); writer.close(); dir.close(); diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java index d6c5fe812140f..2735c03aeb0da 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java @@ -19,12 +19,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; @@ -61,7 +57,10 @@ public void testVersions() throws Exception { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1)); - assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()), nullValue()); + assertThat( + timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean(), randomBoolean()), + nullValue() + ); Document doc = new Document(); doc.add(new StringField(IdFieldMapper.NAME, "1", Field.Store.YES)); @@ -70,7 +69,10 @@ public void testVersions() throws Exception { doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE))); writer.updateDocument(new Term(IdFieldMapper.NAME, "1"), doc); directoryReader = reopen(directoryReader); - assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(1L)); + assertThat( + timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean(), randomBoolean()).version, + equalTo(1L) + ); doc = new Document(); Field uid = new StringField(IdFieldMapper.NAME, "1", Field.Store.YES); @@ -81,7 +83,10 @@ public void testVersions() throws Exception { doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE))); writer.updateDocument(new Term(IdFieldMapper.NAME, "1"), doc); directoryReader = reopen(directoryReader); - assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(2L)); + assertThat( + timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean(), randomBoolean()).version, + equalTo(2L) + ); // test reuse of uid field doc = new Document(); @@ -93,11 +98,17 @@ public void testVersions() throws Exception { writer.updateDocument(new Term(IdFieldMapper.NAME, "1"), doc); directoryReader = reopen(directoryReader); - assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(3L)); + assertThat( + timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean(), randomBoolean()).version, + equalTo(3L) + ); writer.deleteDocuments(new Term(IdFieldMapper.NAME, "1")); directoryReader = reopen(directoryReader); - assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()), nullValue()); + assertThat( + timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean(), randomBoolean()), + nullValue() + ); directoryReader.close(); writer.close(); dir.close(); @@ -125,18 +136,27 @@ public void testNestedDocuments() throws IOException { writer.updateDocuments(new Term(IdFieldMapper.NAME, "1"), docs); DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1)); - assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(5L)); + assertThat( + timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean(), randomBoolean()).version, + equalTo(5L) + ); version.setLongValue(6L); writer.updateDocuments(new Term(IdFieldMapper.NAME, "1"), docs); version.setLongValue(7L); writer.updateDocuments(new Term(IdFieldMapper.NAME, "1"), docs); directoryReader = reopen(directoryReader); - assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(7L)); + assertThat( + timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean(), randomBoolean()).version, + equalTo(7L) + ); writer.deleteDocuments(new Term(IdFieldMapper.NAME, "1")); directoryReader = reopen(directoryReader); - assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()), nullValue()); + assertThat( + timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean(), randomBoolean()), + nullValue() + ); directoryReader.close(); writer.close(); dir.close(); @@ -156,10 +176,16 @@ public void testCache() throws Exception { writer.addDocument(doc); DirectoryReader reader = DirectoryReader.open(writer); // should increase cache size by 1 - assertEquals(87, timeSeriesLoadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version); + assertEquals( + 87, + timeSeriesLoadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean(), randomBoolean()).version + ); assertEquals(size + 1, VersionsAndSeqNoResolver.lookupStates.size()); // should be cache hit - assertEquals(87, timeSeriesLoadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version); + assertEquals( + 87, + timeSeriesLoadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean(), randomBoolean()).version + ); assertEquals(size + 1, VersionsAndSeqNoResolver.lookupStates.size()); reader.close(); @@ -182,11 +208,17 @@ public void testCacheFilterReader() throws Exception { doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE))); writer.addDocument(doc); DirectoryReader reader = DirectoryReader.open(writer); - assertEquals(87, timeSeriesLoadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version); + assertEquals( + 87, + timeSeriesLoadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean(), randomBoolean()).version + ); assertEquals(size + 1, VersionsAndSeqNoResolver.lookupStates.size()); // now wrap the reader DirectoryReader wrapped = ElasticsearchDirectoryReader.wrap(reader, new ShardId("bogus", "_na_", 5)); - assertEquals(87, timeSeriesLoadDocIdAndVersion(wrapped, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version); + assertEquals( + 87, + timeSeriesLoadDocIdAndVersion(wrapped, new Term(IdFieldMapper.NAME, "6"), randomBoolean(), randomBoolean()).version + ); // same size map: core cache key is shared assertEquals(size + 1, VersionsAndSeqNoResolver.lookupStates.size()); @@ -249,14 +281,6 @@ public void testTimeSeriesLoadDocIdAndVersion() throws Exception { } private static String createTSDBId(long timestamp) { - Settings.Builder b = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field"); - IndexMetadata indexMetadata = IndexMetadata.builder("idx").settings(b).numberOfShards(1).numberOfReplicas(0).build(); - IndexRouting.ExtractFromSource.Builder routingBuilder = ((IndexRouting.ExtractFromSource) IndexRouting.fromIndexMetadata( - indexMetadata - )).builder(); - routingBuilder.addMatching("field", new BytesRef("value")); - return createId(false, routingBuilder, new BytesRef("tsid"), timestamp, new byte[16]); + return createId(0, 1, new BytesRef("tsid"), timestamp); } } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index fc29d13667d33..6118a84814462 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -174,7 +174,7 @@ public void testRoutingPathEqualsObjectNameError() { assertThat( e.getMessage(), equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [dim.o] was [object]." ) @@ -192,29 +192,21 @@ public void testRoutingPathMatchesNonDimensionKeyword() { assertThat( e.getMessage(), equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [dim.non_dim] was not a dimension." ) ); } - public void testRoutingPathMatchesNonKeyword() { + public void testRoutingPathMatchesNonKeyword() throws IOException { Settings s = getSettings(randomBoolean() ? "dim.non_kwd" : "dim.*"); - Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(s, mapping(b -> { + createMapperService(s, mapping(b -> { b.startObject("dim").startObject("properties"); b.startObject("non_kwd").field("type", "integer").field("time_series_dimension", true).endObject(); b.startObject("dim").field("type", "keyword").field("time_series_dimension", true).endObject(); b.endObject().endObject(); - }))); - assertThat( - e.getMessage(), - equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [dim.non_kwd] was [integer]." - ) - ); + })); } public void testRoutingPathMatchesScriptedKeyword() { @@ -229,7 +221,7 @@ public void testRoutingPathMatchesScriptedKeyword() { assertThat( e.getMessage(), equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [dim.kwd] has a [script] parameter." ) @@ -245,7 +237,7 @@ public void testRoutingPathMatchesRuntimeKeyword() { assertThat( e.getMessage(), equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [dim.kwd] was a runtime [keyword]." ) diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 1746d91fc20af..a637f340a54f7 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -1602,7 +1602,9 @@ public void testLookupVersionWithPrunedAwayIds() throws IOException { writer.forceMerge(1); try (DirectoryReader reader = DirectoryReader.open(writer)) { assertEquals(1, reader.leaves().size()); - assertNull(VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "1"), false)); + assertNull( + VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "1"), false, false) + ); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index 72583b9aeb19e..3a714fca8fcce 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -70,7 +70,7 @@ private void withIndex(CheckedConsumer buildSource throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "text"))); withLuceneIndex(mapperService, writer -> { - ParsedDocument parsed = mapperService.documentParser().parseDocument(source(buildSource), mapperService.mappingLookup()); + ParsedDocument parsed = mapperService.documentParser().parseDocument(source(buildSource), mapperService.mappingLookup(), 0); writer.addDocuments(parsed.docs()); }, reader -> { assertThat(reader.leaves(), hasSize(1)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index 5945e5c81856f..cd8a0bd2e943b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -46,11 +46,12 @@ public class IdLoaderTests extends ESTestCase { - public void testSynthesizeIdSimple() throws Exception { - var routingPaths = List.of("dim1"); - var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); + private static IdLoader createTsIdLoader() { + return IdLoader.createTsIdLoader(null, null, 7, 1); + } + public void testSynthesizeIdSimple() throws Exception { + var idLoader = createTsIdLoader(); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs = List.of( new Doc(startTime, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), @@ -63,17 +64,17 @@ public void testSynthesizeIdSimple() throws Exception { assertThat(leafReader.numDocs(), equalTo(3)); var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 1, 2 }); // NOTE: time series data is ordered by (tsid, timestamp) - assertThat(leaf.getId(0), equalTo(expectedId(routing, docs.get(2)))); - assertThat(leaf.getId(1), equalTo(expectedId(routing, docs.get(0)))); - assertThat(leaf.getId(2), equalTo(expectedId(routing, docs.get(1)))); + assertThat(leaf.getId(0), equalTo(expectedId(docs.get(2)))); + assertThat(leaf.getId(1), equalTo(expectedId(docs.get(0)))); + assertThat(leaf.getId(2), equalTo(expectedId(docs.get(1)))); }; - prepareIndexReader(indexAndForceMerge(routing, docs), verify, false); + prepareIndexReader(indexAndForceMerge(docs), verify, false); } public void testSynthesizeIdMultipleSegments() throws Exception { var routingPaths = List.of("dim1"); var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); + var idLoader = createTsIdLoader(); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs1 = List.of( @@ -96,15 +97,15 @@ public void testSynthesizeIdMultipleSegments() throws Exception { ); CheckedConsumer buildIndex = writer -> { for (Doc doc : docs1) { - indexDoc(routing, writer, doc); + indexDoc(writer, doc); } writer.flush(); for (Doc doc : docs2) { - indexDoc(routing, writer, doc); + indexDoc(writer, doc); } writer.flush(); for (Doc doc : docs3) { - indexDoc(routing, writer, doc); + indexDoc(writer, doc); } writer.flush(); }; @@ -115,22 +116,22 @@ public void testSynthesizeIdMultipleSegments() throws Exception { assertThat(leafReader.numDocs(), equalTo(docs1.size())); var leaf = idLoader.leaf(null, leafReader, IntStream.range(0, docs1.size()).toArray()); for (int i = 0; i < docs1.size(); i++) { - assertThat(leaf.getId(i), equalTo(expectedId(routing, docs1.get(i)))); + assertThat(leaf.getId(i), equalTo(expectedId(docs1.get(i)))); } } { LeafReader leafReader = indexReader.leaves().get(1).reader(); assertThat(leafReader.numDocs(), equalTo(docs2.size())); var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 3 }); - assertThat(leaf.getId(0), equalTo(expectedId(routing, docs2.get(0)))); - assertThat(leaf.getId(3), equalTo(expectedId(routing, docs2.get(3)))); + assertThat(leaf.getId(0), equalTo(expectedId(docs2.get(0)))); + assertThat(leaf.getId(3), equalTo(expectedId(docs2.get(3)))); } { LeafReader leafReader = indexReader.leaves().get(2).reader(); assertThat(leafReader.numDocs(), equalTo(docs3.size())); var leaf = idLoader.leaf(null, leafReader, new int[] { 1, 2 }); - assertThat(leaf.getId(1), equalTo(expectedId(routing, docs3.get(1)))); - assertThat(leaf.getId(2), equalTo(expectedId(routing, docs3.get(2)))); + assertThat(leaf.getId(1), equalTo(expectedId(docs3.get(1)))); + assertThat(leaf.getId(2), equalTo(expectedId(docs3.get(2)))); } { LeafReader leafReader = indexReader.leaves().get(2).reader(); @@ -145,13 +146,14 @@ public void testSynthesizeIdMultipleSegments() throws Exception { public void testSynthesizeIdRandom() throws Exception { var routingPaths = List.of("dim1"); var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); + var idLoader = createTsIdLoader(); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); Set expectedIDs = new HashSet<>(); List randomDocs = new ArrayList<>(); int numberOfTimeSeries = randomIntBetween(8, 64); for (int i = 0; i < numberOfTimeSeries; i++) { + long routingId = 0; int numberOfDimensions = randomIntBetween(1, 6); List dimensions = new ArrayList<>(numberOfDimensions); for (int j = 1; j <= numberOfDimensions; j++) { @@ -163,12 +165,13 @@ public void testSynthesizeIdRandom() throws Exception { value = randomAlphaOfLength(4); } dimensions.add(new Dimension(fieldName, value)); + routingId = value.hashCode(); } int numberOfSamples = randomIntBetween(1, 16); for (int j = 0; j < numberOfSamples; j++) { Doc doc = new Doc(startTime++, dimensions); randomDocs.add(doc); - expectedIDs.add(expectedId(routing, doc)); + expectedIDs.add(expectedId(doc)); } } CheckedConsumer verify = indexReader -> { @@ -181,14 +184,14 @@ public void testSynthesizeIdRandom() throws Exception { assertTrue("docId=" + i + " id=" + actualId, expectedIDs.remove(actualId)); } }; - prepareIndexReader(indexAndForceMerge(routing, randomDocs), verify, false); + prepareIndexReader(indexAndForceMerge(randomDocs), verify, false); assertThat(expectedIDs, empty()); } - private static CheckedConsumer indexAndForceMerge(IndexRouting.ExtractFromSource routing, List docs) { + private static CheckedConsumer indexAndForceMerge(List docs) { return writer -> { for (Doc doc : docs) { - indexDoc(routing, writer, doc); + indexDoc(writer, doc); } writer.forceMerge(1); }; @@ -220,8 +223,8 @@ private void prepareIndexReader( } } - private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter iw, Doc doc) throws IOException { - final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routing.builder()); + private static void indexDoc(IndexWriter iw, Doc doc) throws IOException { + final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); final List fields = new ArrayList<>(); fields.add(new SortedNumericDocValuesField(DataStreamTimestampFieldMapper.DEFAULT_PATH, doc.timestamp)); @@ -240,9 +243,8 @@ private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter iw.addDocument(fields); } - private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc) throws IOException { - var routingBuilder = routing.builder(); - var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routingBuilder); + private static String expectedId(Doc doc) throws IOException { + var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); for (Dimension dimension : doc.dimensions) { if (dimension.value instanceof Number n) { timeSeriesIdBuilder.addLong(dimension.field, n.longValue()); @@ -250,13 +252,7 @@ private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc timeSeriesIdBuilder.addString(dimension.field, dimension.value.toString()); } } - return TsidExtractingIdFieldMapper.createId( - false, - routingBuilder, - timeSeriesIdBuilder.buildTsidHash().toBytesRef(), - doc.timestamp, - new byte[16] - ); + return TsidExtractingIdFieldMapper.createId(7, 1, timeSeriesIdBuilder.buildTsidHash().toBytesRef(), doc.timestamp); } private static IndexRouting.ExtractFromSource createRouting(List routingPaths) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 542d0088f2ad0..70e375a89d5e7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -601,15 +601,6 @@ protected boolean dedupAfterFetch() { return true; } - @Override - protected String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { - return "All fields that match routing_path must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + mapper.name() - + "] was not a dimension."; - } - public void testDimensionInRoutingPath() throws IOException { MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "keyword").field("time_series_dimension", true))); IndexSettings settings = createIndexSettings( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 94a0f2296bbfb..4e86c1cc1833f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -687,7 +687,8 @@ public void testParseWithDynamicMapping() { false, UNASSIGNED_SEQ_NO, 0, - System.nanoTime() + System.nanoTime(), + 0 ); assertNotNull(index.parsedDoc().dynamicMappingsUpdate()); } @@ -713,13 +714,15 @@ public void testParseWithDynamicMapping() { false, UNASSIGNED_SEQ_NO, 0, - System.nanoTime() + System.nanoTime(), + 0 ); }); assertThat( failure.getMessage(), equalTo( - "[5:1] failed to parse: _id must be unset or set to [AAAAAMpxfIC8Wpr0AAABdrs-cAA]" + "[5:1] failed to parse: _id must be unset or set to " + + "[AAAAAAAAAXa7PnAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA]" + " but was [no-such-tsid] because [index] is in time_series mode" ) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index c19c21d54a569..885b266cc9d4b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -31,6 +32,9 @@ import static org.hamcrest.Matchers.equalTo; public class TsidExtractingIdFieldMapperTests extends MetadataMapperTestCase { + + private static final int SHARD_ID = 7; + private static class TestCase { private final String name; private final String expectedId; @@ -82,7 +86,7 @@ public static Iterable params() { items.add( new TestCase( "2022-01-01T01:00:00Z", - "XsFI2ajcFfi45iV3AAABfhMmioA", + "BwAAAAAAAX4TJoqAJJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { @@ -94,7 +98,7 @@ public static Iterable params() { items.add( new TestCase( "2022-01-01T01:00:01Z", - "XsFI2ajcFfi45iV3AAABfhMmjmg", + "BwAAAAAAAX4TJo5oJJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:01.000Z", b -> { @@ -106,7 +110,7 @@ public static Iterable params() { items.add( new TestCase( "1970-01-01T00:00:00Z", - "XsFI2ajcFfi45iV3AAAAAAAAAAA", + "BwAAAAAAAAAAAAAAJJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "1970-01-01T00:00:00.000Z", b -> { @@ -118,7 +122,7 @@ public static Iterable params() { items.add( new TestCase( "-9998-01-01T00:00:00Z", - "XsFI2ajcFfi45iV3__6oggRgGAA", + "BwAAAP_-qIIEYBgAJJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "-9998-01-01T00:00:00.000Z", b -> { @@ -130,7 +134,7 @@ public static Iterable params() { items.add( new TestCase( "9998-01-01T00:00:00Z", - "XsFI2ajcFfi45iV3AADmaSK9hAA", + "BwAAAAAA5mkivYQAJJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "9998-01-01T00:00:00.000Z", b -> { @@ -144,7 +148,7 @@ public static Iterable params() { items.add( new TestCase( "r1", - "XsFI2ajcFfi45iV3AAABfhMmioA", + "BwAAAAAAAX4TJoqAJJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { @@ -180,7 +184,7 @@ public static Iterable params() { items.add( new TestCase( "r2", - "1y-UzR0iuE1-sOQpAAABfhMmioA", + "BwAAAAAAAX4TJoqAJNY_frTR9GmCbhXgK4Y8W44GlT_5e6_NYGOZWULpmMG9IAlZlA", "JNY_frTR9GmCbhXgK4Y8W44GlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { @@ -192,7 +196,7 @@ public static Iterable params() { items.add( new TestCase( "o.r3", - "zh4dcS1h1gf2J5a8AAABfhMmioA", + "BwAAAAAAAX4TJoqAJEyfZsJIp3UNyfWG-4SjKFIGlT_5e6_NYGOZWULpmMG9IAlZlA", "JEyfZsJIp3UNyfWG-4SjKFIGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { @@ -209,7 +213,7 @@ public static Iterable params() { items.add( new TestCase( "k1=dog", - "XsFI2SrEiVgZlSsYAAABfhMmioA", + "BwAAAAAAAX4TJoqAKJQKpjU9U63jhh-eNJ1f8bipyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "KJQKpjU9U63jhh-eNJ1f8bipyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { @@ -222,7 +226,7 @@ public static Iterable params() { items.add( new TestCase( "k1=pumpkin", - "XsFI2W8GX8-0QcFxAAABfhMmioA", + "BwAAAAAAAX4TJoqAKJQKpjU9U63jhh-eNJ1f8bibzw1JBpU_-VsHjSz5HC1yy_swPEM1iGo", "KJQKpjU9U63jhh-eNJ1f8bibzw1JBpU_-VsHjSz5HC1yy_swPEM1iGo", "2022-01-01T01:00:00.000Z", b -> { @@ -235,7 +239,7 @@ public static Iterable params() { items.add( new TestCase( "k1=empty string", - "XsFI2cna58i6D-Q6AAABfhMmioA", + "BwAAAAAAAX4TJoqAKJQKpjU9U63jhh-eNJ1f8bhaCD7uBpU_-SWGG0Uv9tZ1mLO2gi9rC1I", "KJQKpjU9U63jhh-eNJ1f8bhaCD7uBpU_-SWGG0Uv9tZ1mLO2gi9rC1I", "2022-01-01T01:00:00.000Z", b -> { @@ -248,7 +252,7 @@ public static Iterable params() { items.add( new TestCase( "k2", - "XsFI2VqlzAuv-06kAAABfhMmioA", + "BwAAAAAAAX4TJoqAKB9H-tGrL_UzqMcqXcgBtzypyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "KB9H-tGrL_UzqMcqXcgBtzypyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { @@ -261,7 +265,7 @@ public static Iterable params() { items.add( new TestCase( "o.k3", - "XsFI2S_VhridAKDUAAABfhMmioA", + "BwAAAAAAAX4TJoqAKGXATwN7ISd1_EycFRJ9h6qpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "KGXATwN7ISd1_EycFRJ9h6qpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { @@ -274,7 +278,7 @@ public static Iterable params() { items.add( new TestCase( "o.r3", - "zh4dcUwfL7x__2oPAAABfhMmioA", + "BwAAAAAAAX4TJoqAKJaYZVZz8plfkEvvPBpi1EWpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "KJaYZVZz8plfkEvvPBpi1EWpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { @@ -305,7 +309,7 @@ public static Iterable params() { items.add( new TestCase( "L1=1", - "XsFI2fIe53BtV9PCAAABfhMmioA", + "BwAAAAAAAX4TJoqAKI4kVxcCLIMM2_VQGD575d-tm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "KI4kVxcCLIMM2_VQGD575d-tm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "2022-01-01T01:00:00.000Z", b -> { @@ -318,7 +322,7 @@ public static Iterable params() { items.add( new TestCase( "L1=min", - "XsFI2Qhu7hy1RoXRAAABfhMmioA", + "BwAAAAAAAX4TJoqAKI4kVxcCLIMM2_VQGD575d8caJ3TBpU_-cLpg-VnCBnhYk33HZBle6E", "KI4kVxcCLIMM2_VQGD575d8caJ3TBpU_-cLpg-VnCBnhYk33HZBle6E", "2022-01-01T01:00:00.000Z", b -> { @@ -331,7 +335,7 @@ public static Iterable params() { items.add( new TestCase( "L2=1234", - "XsFI2QTrNu7TTpc-AAABfhMmioA", + "BwAAAAAAAX4TJoqAKI_1WxF60L0IczG5ftUCWdndcGtgBpU_-QfM2BaR0DMagIfw3TDu_mA", "KI_1WxF60L0IczG5ftUCWdndcGtgBpU_-QfM2BaR0DMagIfw3TDu_mA", "2022-01-01T01:00:00.000Z", b -> { @@ -344,7 +348,7 @@ public static Iterable params() { items.add( new TestCase( "o.L3=max", - "zh4dcWBQI6THHqxoAAABfhMmioA", + "BwAAAAAAAX4TJoqAKN4a6QzKhzc3nwzNLuZkV51xxTOVBpU_-erUU1qSW4eJ0kP0RmAB9TE", "KN4a6QzKhzc3nwzNLuZkV51xxTOVBpU_-erUU1qSW4eJ0kP0RmAB9TE", "2022-01-01T01:00:00.000Z", b -> { @@ -375,7 +379,7 @@ public static Iterable params() { items.add( new TestCase( "i1=1", - "XsFI2UMS_RWRoHYjAAABfhMmioA", + "BwAAAAAAAX4TJoqAKLGFpvAV8QkWSmX54kXFMgitm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "KLGFpvAV8QkWSmX54kXFMgitm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "2022-01-01T01:00:00.000Z", b -> { @@ -388,7 +392,7 @@ public static Iterable params() { items.add( new TestCase( "i1=min", - "XsFI2adlQM5ILoA1AAABfhMmioA", + "BwAAAAAAAX4TJoqAKLGFpvAV8QkWSmX54kXFMgjV8hFQBpU_-WG2MicRGWwJdBKWq2F4qy4", "KLGFpvAV8QkWSmX54kXFMgjV8hFQBpU_-WG2MicRGWwJdBKWq2F4qy4", "2022-01-01T01:00:00.000Z", b -> { @@ -401,7 +405,7 @@ public static Iterable params() { items.add( new TestCase( "i2=1234", - "XsFI2bhxfB6J0kBFAAABfhMmioA", + "BwAAAAAAAX4TJoqAKJc4-5eN1uAlYuAknQQLUlxavn2sBpU_-UEXBjgaH1uYcbayrOhdgpc", "KJc4-5eN1uAlYuAknQQLUlxavn2sBpU_-UEXBjgaH1uYcbayrOhdgpc", "2022-01-01T01:00:00.000Z", b -> { @@ -414,7 +418,7 @@ public static Iterable params() { items.add( new TestCase( "o.i3=max", - "zh4dcelxKf19CbfdAAABfhMmioA", + "BwAAAAAAAX4TJoqAKKqnzPNBe8ObksSo8rNaIFPZPCcBBpU_-Rhd_U6Jn2pjQz2zpmBuJb4", "KKqnzPNBe8ObksSo8rNaIFPZPCcBBpU_-Rhd_U6Jn2pjQz2zpmBuJb4", "2022-01-01T01:00:00.000Z", b -> { @@ -445,7 +449,7 @@ public static Iterable params() { items.add( new TestCase( "s1=1", - "XsFI2Y_y-8kD_BFeAAABfhMmioA", + "BwAAAAAAAX4TJoqAKFi_JDbvzWyAawmh8IEXedwGlT_5rZuNb-1ruHTTZhtsXRZpZRwWFoc", "KFi_JDbvzWyAawmh8IEXedwGlT_5rZuNb-1ruHTTZhtsXRZpZRwWFoc", "2022-01-01T01:00:00.000Z", b -> { @@ -458,7 +462,7 @@ public static Iterable params() { items.add( new TestCase( "s1=min", - "XsFI2WV8VNVnmPVNAAABfhMmioA", + "BwAAAAAAAX4TJoqAKFi_JDbvzWyAawmh8IEXedwGlT_5JgBZj9BSCms2_jgeFFhsmDlNFdM", "KFi_JDbvzWyAawmh8IEXedwGlT_5JgBZj9BSCms2_jgeFFhsmDlNFdM", "2022-01-01T01:00:00.000Z", b -> { @@ -471,7 +475,7 @@ public static Iterable params() { items.add( new TestCase( "s2=1234", - "XsFI2VO8mUr-J5CpAAABfhMmioA", + "BwAAAAAAAX4TJoqAKKEQ2p3CkpMH61hNk_SuvI0GlT_53XBrYP5TPdmCR-vREPnt20e9f9w", "KKEQ2p3CkpMH61hNk_SuvI0GlT_53XBrYP5TPdmCR-vREPnt20e9f9w", "2022-01-01T01:00:00.000Z", b -> { @@ -484,7 +488,7 @@ public static Iterable params() { items.add( new TestCase( "o.s3=max", - "zh4dcQKh6K11zWeuAAABfhMmioA", + "BwAAAAAAAX4TJoqAKKVMoT_-GS95fvIBtR7XK9oGlT_5Dme9-H3sen0WZ7leJpCj7-vXau4", "KKVMoT_-GS95fvIBtR7XK9oGlT_5Dme9-H3sen0WZ7leJpCj7-vXau4", "2022-01-01T01:00:00.000Z", b -> { @@ -515,7 +519,7 @@ public static Iterable params() { items.add( new TestCase( "b1=1", - "XsFI2dKxqgT5JDQfAAABfhMmioA", + "BwAAAAAAAX4TJoqAKGPAUhTjWOsRfDmYp3SUELatm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "KGPAUhTjWOsRfDmYp3SUELatm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "2022-01-01T01:00:00.000Z", b -> { @@ -528,7 +532,7 @@ public static Iterable params() { items.add( new TestCase( "b1=min", - "XsFI2d_PD--DgUvoAAABfhMmioA", + "BwAAAAAAAX4TJoqAKGPAUhTjWOsRfDmYp3SUELYoK6qHBpU_-d8HkZFJ3aL2ZV1lgHAjT1g", "KGPAUhTjWOsRfDmYp3SUELYoK6qHBpU_-d8HkZFJ3aL2ZV1lgHAjT1g", "2022-01-01T01:00:00.000Z", b -> { @@ -541,7 +545,7 @@ public static Iterable params() { items.add( new TestCase( "b2=12", - "XsFI2aqX5QjiuhsEAAABfhMmioA", + "BwAAAAAAAX4TJoqAKA58oUMzXeX1V5rh51Ste0K5K9vPBpU_-Wn8JQplO-x3CgoslYO5Vks", "KA58oUMzXeX1V5rh51Ste0K5K9vPBpU_-Wn8JQplO-x3CgoslYO5Vks", "2022-01-01T01:00:00.000Z", b -> { @@ -554,7 +558,7 @@ public static Iterable params() { items.add( new TestCase( "o.s3=max", - "zh4dccJ4YtN_21XHAAABfhMmioA", + "BwAAAAAAAX4TJoqAKIwZH-StJBobjk9tCV-0OgjKmuwGBpU_-Sd-SdnoH3sbfKLgse-briE", "KIwZH-StJBobjk9tCV-0OgjKmuwGBpU_-Sd-SdnoH3sbfKLgse-briE", "2022-01-01T01:00:00.000Z", b -> { @@ -585,7 +589,7 @@ public static Iterable params() { items.add( new TestCase( "ip1=192.168.0.1", - "XsFI2T5km9raIz_rAAABfhMmioA", + "BwAAAAAAAX4TJoqAKNj6cLPRNEkqdjfOPIbg0wULrOlWBpU_-efWDsz6B6AnnwbZ7GeeocE", "KNj6cLPRNEkqdjfOPIbg0wULrOlWBpU_-efWDsz6B6AnnwbZ7GeeocE", "2022-01-01T01:00:00.000Z", b -> { @@ -602,7 +606,7 @@ public static Iterable params() { items.add( new TestCase( "ip1=12.12.45.254", - "XsFI2QWfEH_e_6wIAAABfhMmioA", + "BwAAAAAAAX4TJoqAKNj6cLPRNEkqdjfOPIbg0wVhJ08TBpU_-bANzLhvKPczlle7Pq0z8Qw", "KNj6cLPRNEkqdjfOPIbg0wVhJ08TBpU_-bANzLhvKPczlle7Pq0z8Qw", "2022-01-01T01:00:00.000Z", b -> { @@ -619,7 +623,7 @@ public static Iterable params() { items.add( new TestCase( "ip2=FE80:CD00:0000:0CDE:1257:0000:211E:729C", - "XsFI2WrrLHr1O4iQAAABfhMmioA", + "BwAAAAAAAX4TJoqAKNDo3zGxO9HfN9XYJwKw2Z20h-WsBpU_-f4dSOLGSRlL1hoY2mgERuo", "KNDo3zGxO9HfN9XYJwKw2Z20h-WsBpU_-f4dSOLGSRlL1hoY2mgERuo", "2022-01-01T01:00:00.000Z", b -> { @@ -632,7 +636,7 @@ public static Iterable params() { items.add( new TestCase( "o.ip3=2001:db8:85a3:8d3:1319:8a2e:370:7348", - "zh4dca7d-9aKOS1MAAABfhMmioA", + "BwAAAAAAAX4TJoqAKLXDcBBWJAjgJvjSdF_EJwraAQUzBpU_-ba6HZsIyKnGcbmc3KRLlmI", "KLXDcBBWJAjgJvjSdF_EJwraAQUzBpU_-ba6HZsIyKnGcbmc3KRLlmI", "2022-01-01T01:00:00.000Z", b -> { @@ -663,7 +667,7 @@ public static Iterable params() { items.add( new TestCase( "huge", - "WZKJR_dECvXBSl3xAAABfhMmioA", + "BwAAAAAAAX4TJoqALIe18i0rRU_Bt9vB82F46LaS9mrUkvZq1K_2Gi7UEFMhFwNXrLA_H8TLpUr4", "LIe18i0rRU_Bt9vB82F46LaS9mrUkvZq1K_2Gi7UEFMhFwNXrLA_H8TLpUr4", "2022-01-01T01:00:00.000Z", b -> { @@ -721,16 +725,15 @@ private ParsedDocument parse(@Nullable String id, MapperService mapperService, C source.accept(builder); builder.endObject(); SourceToParse sourceToParse = new SourceToParse(id, BytesReference.bytes(builder), builder.contentType()); - return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup()); + return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup(), SHARD_ID); } } public void testRoutingPathCompliant() throws IOException { - IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random()); + IndexVersion version = IndexVersions.TIME_SERIES_SHARD_ID_IN_ID; IndexRouting indexRouting = createIndexSettings(version, indexSettings(version)).getIndexRouting(); - int indexShard = indexShard(indexRouting); - assertThat(indexRouting.getShard(testCase.expectedId, null), equalTo(indexShard)); - assertThat(indexRouting.deleteShard(testCase.expectedId, null), equalTo(indexShard)); + assertThat(indexRouting.getShard(testCase.expectedId, null), equalTo(SHARD_ID)); + assertThat(indexRouting.deleteShard(testCase.expectedId, null), equalTo(SHARD_ID)); } private int indexShard(IndexRouting indexRouting) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index e4ea78f3b7a0e..4f23c86f53cca 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -467,9 +467,9 @@ public void testMissingDimensionInRoutingPath() throws IOException { ); Exception ex = expectThrows(IllegalArgumentException.class, () -> mapper.documentMapper().validate(settings, false)); assertEquals( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [field.key3] was [flattened].", + + "without the [script] parameter. [field._keyed] was not a dimension.", ex.getMessage() ); } diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 82fb694db6c66..d4af486f34933 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -497,7 +497,8 @@ public Engine.Index createIndexOp(int docIdent) { false, UNASSIGNED_SEQ_NO, 0, - System.nanoTime() + System.nanoTime(), + 0 ); } } @@ -534,7 +535,8 @@ public Engine.Index createIndexOp(int docIdent) { false, UNASSIGNED_SEQ_NO, 0, - System.nanoTime() + System.nanoTime(), + 0 ); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index 423990999fabd..d3e3331b3dc3d 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -106,7 +106,8 @@ public Engine.Operation convertToEngineOp(Translog.Operation operation, Engine.O true, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, - System.nanoTime() + System.nanoTime(), + 0 ); return engineIndex; } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 43ac8057a3fc0..4bc9b10f1a764 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1030,8 +1030,17 @@ public final void testMinimalIsInvalidInRoutingPath() throws IOException { } } - protected String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { - return "All fields that match routing_path must be keywords with [time_series_dimension: true] " + private String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { + if (mapper instanceof FieldMapper fieldMapper + && fieldMapper.fieldType().supportsDimension() + && fieldMapper.fieldType().isDimension() == false) { + return "All fields that match routing_path must be configured with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + mapper.name() + + "] was not a dimension."; + } + return "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [" + mapper.name() diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 81848b5a50114..c60a913a63b33 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -253,12 +253,8 @@ public void testDimension() throws IOException { // dimension = false is allowed assertDimension(false, NumberFieldMapper.NumberFieldType::isDimension); - // dimension = true is not allowed - Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_dimension", true); - }))); - assertThat(e.getCause().getMessage(), containsString("Parameter [time_series_dimension] cannot be set")); + // dimension = true is allowed + assertDimension(true, NumberFieldMapper.NumberFieldType::isDimension); } public void testMetricType() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index d4c238322e28a..b5f59fc130977 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -67,7 +67,8 @@ private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse sou ), source, mappingLookup.getMapping().getRoot(), - ObjectMapper.Dynamic.getRootDynamic(mappingLookup) + ObjectMapper.Dynamic.getRootDynamic(mappingLookup), + 0 ); this.parser = parser; } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java index 1ba9fa5d1d354..ddec29eaf95f0 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java @@ -352,7 +352,8 @@ private IndexReader initIndex(Directory directory, int size, int commitEvery) th ParsedDocument doc = mapperService.documentParser() .parseDocument( new SourceToParse("id" + d, BytesReference.bytes(source), XContentType.JSON), - mapperService.mappingLookup() + mapperService.mappingLookup(), + 0 ); writer.addDocuments(doc.docs()); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 52424956ef53e..7c7b30f0bf7f3 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -76,7 +76,7 @@ private static UnsignedLongFieldMapper toType(FieldMapper in) { return (UnsignedLongFieldMapper) in; } - public static final class Builder extends FieldMapper.Builder { + public static final class Builder extends FieldMapper.DimensionBuilder { private final Parameter indexed; private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); private final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); @@ -195,7 +195,7 @@ Number parsedNullValue() { @Override public UnsignedLongFieldMapper build(MapperBuilderContext context) { - if (context.parentObjectContainsDimensions()) { + if (super.isDimension || context.parentObjectContainsDimensions()) { dimension.setValue(true); } UnsignedLongFieldType fieldType = new UnsignedLongFieldType( @@ -539,13 +539,16 @@ static Long parseUpperRangeTerm(Object value, boolean include) { return longValue; } - /** - * @return true if field has been marked as a dimension field - */ + @Override public boolean isDimension() { return isDimension; } + @Override + public boolean supportsDimension() { + return true; + } + /** * If field is a time series metric field, returns its metric type * @return the metric type or null