From 37ae32aecd25bbbdfcce8ded6e372b5f957ab401 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 4 Aug 2025 15:19:28 +0200 Subject: [PATCH 01/62] Hash once to create routing hash and _tsid --- gradle/verification-metadata.xml | 5 + .../DataStreamIndexSettingsProvider.java | 41 ++-- .../test/data_stream/150_tsdb.yml | 19 +- server/build.gradle | 1 + server/licenses/hash4j-LICENSE.txt | 201 ++++++++++++++++ server/src/main/java/module-info.java | 1 + .../org/elasticsearch/TransportVersions.java | 1 + .../action/bulk/TransportShardBulkAction.java | 9 +- .../bulk/TransportSimulateBulkAction.java | 3 +- .../action/index/IndexRequest.java | 30 ++- .../action/index/IndexRequestBuilder.java | 10 + .../cluster/metadata/IndexMetadata.java | 20 ++ .../cluster/routing/IndexRouting.java | 220 ++++++++++++++---- .../cluster/routing/TsidBuilder.java | 192 +++++++++++++++ .../common/settings/IndexScopedSettings.java | 1 + .../org/elasticsearch/index/IndexMode.java | 30 ++- .../elasticsearch/index/IndexVersions.java | 1 + .../index/mapper/DocumentParser.java | 13 +- .../index/mapper/DocumentParserContext.java | 11 +- .../elasticsearch/index/mapper/IdLoader.java | 5 +- .../index/mapper/RoutingFields.java | 11 +- .../index/mapper/RoutingPathFields.java | 11 +- .../index/mapper/SourceToParse.java | 20 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 24 +- .../mapper/TsidExtractingIdFieldMapper.java | 4 +- .../flattened/FlattenedFieldParser.java | 2 +- .../action/index/IndexRequestTests.java | 3 + .../cluster/routing/IndexRoutingTests.java | 146 +++++++++--- .../index/mapper/DynamicTemplatesTests.java | 2 +- .../index/mapper/BlockLoaderTestRunner.java | 3 +- .../index/mapper/MapperServiceTestCase.java | 2 +- .../mapper/TestDocumentParserContext.java | 6 + .../test/InternalTestCluster.java | 2 +- 33 files changed, 898 insertions(+), 152 deletions(-) create mode 100644 server/licenses/hash4j-LICENSE.txt create mode 100644 server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index f8f81bd90eeeb..ae9e87785d0df 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -266,6 +266,11 @@ + + + + + diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index ac3987dfa2771..f69b3e6b90195 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.core.CheckedFunction; @@ -23,6 +24,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; @@ -38,6 +40,7 @@ import java.util.List; import java.util.Locale; +import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_DIMENSIONS; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; /** @@ -117,15 +120,16 @@ public Settings getAdditionalIndexSettings( builder.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), FORMATTER.format(start)); builder.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), FORMATTER.format(end)); - if (indexTemplateAndCreateRequestSettings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey()) == false - && combinedTemplateMappings.isEmpty() == false) { - List routingPaths = findRoutingPaths( + if (combinedTemplateMappings.isEmpty() == false) { + List dimensions = findDimensionFields( indexName, indexTemplateAndCreateRequestSettings, combinedTemplateMappings ); - if (routingPaths.isEmpty() == false) { - builder.putList(INDEX_ROUTING_PATH.getKey(), routingPaths); + if (dimensions.isEmpty() == false) { + // TODO handle the case when adding a dimension field to the mappings of an existing index + // at the moment, the index.dimensions setting is only set when an index is created + builder.putList(INDEX_DIMENSIONS.getKey(), dimensions); } } return builder.build(); @@ -137,15 +141,15 @@ public Settings getAdditionalIndexSettings( } /** - * Find fields in mapping that are of type keyword and time_series_dimension enabled. + * Find fields in mapping that are time_series_dimension enabled. * Using MapperService here has an overhead, but allows the mappings from template to * be merged correctly and fetching the fields without manually parsing the mappings. - * + *

* Alternatively this method can instead parse mappings into map of maps and merge that and * iterate over all values to find the field that can serve as routing value. But this requires * mapping specific logic to exist here. */ - private List findRoutingPaths(String indexName, Settings allSettings, List combinedTemplateMappings) { + private List findDimensionFields(String indexName, Settings allSettings, List combinedTemplateMappings) { var tmpIndexMetadata = IndexMetadata.builder(indexName); int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(allSettings); @@ -170,9 +174,6 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata.build())) { mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); List routingPaths = new ArrayList<>(); - for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) { - extractPath(routingPaths, fieldMapper); - } for (var objectMapper : mapperService.documentMapper().mappers().objectMappers().values()) { if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { if (passThroughObjectMapper.containsDimensions()) { @@ -180,6 +181,9 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li } } } + for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) { + extractPath(routingPaths, fieldMapper); + } for (var template : mapperService.getAllDynamicTemplates()) { if (template.pathMatch().isEmpty()) { continue; @@ -193,7 +197,7 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li } MappingParserContext parserContext = mapperService.parserContext(); - for (Iterator iterator = template.pathMatch().iterator(); iterator.hasNext();) { + for (Iterator iterator = template.pathMatch().iterator(); iterator.hasNext(); ) { var mapper = parserContext.typeParser(mappingSnippetType) .parse(iterator.next(), mappingSnippet, parserContext) .build(MapperBuilderContext.root(false, false)); @@ -214,12 +218,17 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li } /** - * Helper method that adds the name of the mapper to the provided list if it is a keyword dimension field. + * Helper method that adds the name of the mapper to the provided list. */ private static void extractPath(List routingPaths, Mapper mapper) { - if (mapper instanceof KeywordFieldMapper keywordFieldMapper) { - if (keywordFieldMapper.fieldType().isDimension()) { - routingPaths.add(mapper.fullPath()); + if (mapper instanceof FieldMapper fieldMapper) { + if (fieldMapper.fieldType().isDimension()) { + String path = mapper.fullPath(); + // don't add if the path already matches via a wildcard pattern in the list + // e.g. if "path.*" is already added, "path.foo" should not be added + if (Regex.simpleMatch(routingPaths, path) == false) { + routingPaths.add(path); + } } } } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 598bc90217574..f393fafb5067e 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -148,13 +148,12 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" ]} --- "aggregate the tsid": - requires: cluster_features: ["gte_v8.13.0"] - reason: _tsid hahing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: @@ -169,9 +168,8 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} + - length: {aggregations.tsids.buckets: 2} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -368,7 +366,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -387,7 +384,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -406,7 +402,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -425,7 +420,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: search: @@ -443,7 +437,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -520,7 +513,6 @@ dynamic templates - conflicting aliases: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KGejYryCnrIkXYZdIF_Q8F8X2dfFIGKYisFh7t1RGGWOWgWU7C0RiFE" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -539,7 +531,6 @@ dynamic templates - conflicting aliases: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KGejYryCnrIkXYZdIF_Q8F8X2dfFIGKYisFh7t1RGGWOWgWU7C0RiFE" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -699,7 +690,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -718,7 +708,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -737,7 +726,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -756,7 +744,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -775,7 +762,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -794,7 +780,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- diff --git a/server/build.gradle b/server/build.gradle index 59615e5f42ac5..2c2c9771adcb5 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -59,6 +59,7 @@ dependencies { // utilities api project(":libs:cli") implementation 'com.carrotsearch:hppc:0.8.1' + api 'com.dynatrace.hash4j:hash4j:0.25.0' // precentil ranks aggregation api 'org.hdrhistogram:HdrHistogram:2.1.9' diff --git a/server/licenses/hash4j-LICENSE.txt b/server/licenses/hash4j-LICENSE.txt new file mode 100644 index 0000000000000..261eeb9e9f8b2 --- /dev/null +++ b/server/licenses/hash4j-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 90cd3c669a52c..393d180be05f9 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -54,6 +54,7 @@ requires org.apache.lucene.queryparser; requires org.apache.lucene.sandbox; requires org.apache.lucene.suggest; + requires hash4j; exports org.elasticsearch; exports org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index deab86cbbb227..20bd7f357aa6d 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -356,6 +356,7 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_AI21_COMPLETION_ADDED = def(9_134_0_00); public static final TransportVersion TRANSPORT_NODE_USAGE_STATS_FOR_THREAD_POOLS_ACTION = def(9_135_0_00); public static final TransportVersion INDEX_TEMPLATE_TRACKING_INFO = def(9_136_0_00); + public static final TransportVersion INGEST_REQUEST_INCLUDE_TSID = def(9_137_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 01968596db932..357504fe87d1b 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -415,7 +415,8 @@ static boolean executeBulkItemRequest( request.routing(), request.getDynamicTemplates(), request.getIncludeSourceOnError(), - meteringParserDecorator + meteringParserDecorator, + request.tsid() ); result = primary.applyIndexOperationOnPrimary( version, @@ -743,7 +744,11 @@ private static Engine.Result performOpOnReplica( indexRequest.id(), indexRequest.source(), indexRequest.getContentType(), - indexRequest.routing() + indexRequest.routing(), + Map.of(), + true, + XContentMeteringParserDecorator.NOOP, + indexRequest.tsid() ); result = replica.applyIndexOperationOnReplica( primaryResponse.getSeqNo(), diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 85e9f2c5084de..6036c1ff5cbda 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -191,7 +191,8 @@ private Tuple, Exception> validateMappings( request.routing(), request.getDynamicTemplates(), request.getIncludeSourceOnError(), - XContentMeteringParserDecorator.NOOP + XContentMeteringParserDecorator.NOOP, + request.tsid() ); ProjectMetadata project = projectResolver.getProjectMetadata(clusterService.state()); diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 707ea0919a91f..f5051eabca2a7 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.index; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchGenerationException; @@ -152,6 +153,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement * rawTimestamp field is used on the coordinate node, it doesn't need to be serialised. */ private Object rawTimestamp; + private BytesRef tsid; public IndexRequest(StreamInput in) throws IOException { this(null, in); @@ -216,6 +218,13 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio if (in.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR)) { includeSourceOnError = in.readBoolean(); } // else default value is true + + if (in.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_TSID)) { + tsid = in.readBytesRef(); + if (tsid.length == 0) { + tsid = null; // no tsid set + } + } } public IndexRequest() { @@ -353,6 +362,22 @@ public String routing() { return this.routing; } + /** + * For time series indices created after {@link org.elasticsearch.index.IndexVersions#TSID_CREATED_DURING_ROUTING}, + * the coordinating node will calculate _tsid during routing and set it on the request. + * For time series indices created before that version, the _tsid will be created in the data node during document parsing. + *

+ * The _tsid can not be directly set by a user, it is set by the coordinating node. + */ + public IndexRequest tsid(BytesRef tsid) { + this.tsid = tsid; + return this; + } + + public BytesRef tsid() { + return this.tsid; + } + /** * Sets the ingest pipeline to be executed before indexing the document */ @@ -815,6 +840,9 @@ private void writeBody(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR)) { out.writeBoolean(includeSourceOnError); } + if (out.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_TSID)) { + out.writeBytesRef(tsid); + } } @Override @@ -917,7 +945,7 @@ public Index getConcreteWriteIndex(IndexAbstraction ia, ProjectMetadata project) @Override public int route(IndexRouting indexRouting) { - return indexRouting.indexShard(id, routing, contentType, source); + return indexRouting.indexShard(id, routing, tsid, contentType, source); } public IndexRequest setRequireAlias(boolean requireAlias) { diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index 3041e6bf5e274..620cd90cbebfc 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.index; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; @@ -50,6 +51,7 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder> settings() { Property.ServerlessPublic ); + public static final Setting> INDEX_DIMENSIONS = Setting.stringListSetting( + "index.dimensions", + Setting.Property.IndexScope, + Setting.Property.Final + ); + /** * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use. * TODO: Remove in 9.0 @@ -576,6 +582,7 @@ public Iterator> settings() { private final int routingFactor; private final int routingPartitionSize; private final List routingPaths; + private final List dimensions; private final int numberOfShards; private final int numberOfReplicas; @@ -689,6 +696,7 @@ private IndexMetadata( final int routingNumShards, final int routingPartitionSize, final List routingPaths, + final List dimensions, final ActiveShardCount waitForActiveShards, final ImmutableOpenMap rolloverInfos, final boolean isSystem, @@ -744,6 +752,7 @@ private IndexMetadata( this.routingFactor = routingNumShards / numberOfShards; this.routingPartitionSize = routingPartitionSize; this.routingPaths = routingPaths; + this.dimensions = dimensions; this.waitForActiveShards = waitForActiveShards; this.rolloverInfos = rolloverInfos; this.isSystem = isSystem; @@ -803,6 +812,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) { this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -865,6 +875,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set inSyncSet) this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -935,6 +946,7 @@ public IndexMetadata withSetPrimaryTerm(int shardId, long primaryTerm) { this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -996,6 +1008,7 @@ public IndexMetadata withTimestampRanges(IndexLongFieldRange timestampRange, Ind this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -1052,6 +1065,7 @@ public IndexMetadata withIncrementedVersion() { this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -1166,6 +1180,10 @@ public List getRoutingPaths() { return routingPaths; } + public List getDimensions() { + return dimensions; + } + public int getTotalNumberOfShards() { return totalNumberOfShards; } @@ -2378,6 +2396,7 @@ IndexMetadata build(boolean repair) { } final List routingPaths = INDEX_ROUTING_PATH.get(settings); + final List dimensions = INDEX_DIMENSIONS.get(settings); final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE); @@ -2457,6 +2476,7 @@ IndexMetadata build(boolean repair) { getRoutingNumShards(), routingPartitionSize, routingPaths, + dimensions, waitForActiveShards, rolloverInfos.build(), isSystem, diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 050181802af8d..e256edbb97cc4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexMode; @@ -42,7 +43,6 @@ import java.util.Base64; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.OptionalInt; import java.util.Set; import java.util.function.IntConsumer; @@ -63,7 +63,7 @@ public abstract class IndexRouting { * Build the routing from {@link IndexMetadata}. */ public static IndexRouting fromIndexMetadata(IndexMetadata metadata) { - if (false == metadata.getRoutingPaths().isEmpty()) { + if (metadata.getRoutingPaths().isEmpty() == false || metadata.getDimensions().isEmpty() == false) { return new ExtractFromSource(metadata); } if (metadata.isRoutingPartitionedIndex()) { @@ -98,7 +98,13 @@ public void postProcess(IndexRequest indexRequest) {} * Called when indexing a document to generate the shard id that should contain * a document with the provided parameters. */ - public abstract int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source); + public abstract int indexShard( + String id, + @Nullable String routing, + @Nullable BytesRef tsid, + XContentType sourceType, + BytesReference source + ); /** * Called when updating a document to generate the shard id that should contain @@ -211,7 +217,13 @@ private static boolean isNewIndexVersion(final IndexVersion creationVersion) { } @Override - public int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source) { + public int indexShard( + String id, + @Nullable String routing, + @Nullable BytesRef tsid, + XContentType sourceType, + BytesReference source + ) { if (id == null) { throw new IllegalStateException("id is required and should have been set by process"); } @@ -300,8 +312,11 @@ public static class ExtractFromSource extends IndexRouting { private final XContentParserConfiguration parserConfig; private final IndexMode indexMode; private final boolean trackTimeSeriesRoutingHash; + private final boolean createTsidDuringRouting; private final boolean addIdWithRoutingHash; private int hash = Integer.MAX_VALUE; + @Nullable + private BytesRef tsid; ExtractFromSource(IndexMetadata metadata) { super(metadata); @@ -309,12 +324,27 @@ public static class ExtractFromSource extends IndexRouting { throw new IllegalArgumentException("routing_partition_size is incompatible with routing_path"); } indexMode = metadata.getIndexMode(); - trackTimeSeriesRoutingHash = indexMode == IndexMode.TIME_SERIES - && metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID); + var createTsidDuringRouting = false; + var trackTimeSeriesRoutingHash = false; + List includePaths; + includePaths = metadata.getRoutingPaths(); + if (indexMode == IndexMode.TIME_SERIES) { + if (metadata.getDimensions().isEmpty() == false && metadata.getRoutingPaths().isEmpty()) { + // This optimization is only available for new indices where + // the dimensions index setting is automatically populated from the mappings. + // If users manually set the routing paths, the optimization is not applied. + createTsidDuringRouting = true; + includePaths = metadata.getDimensions(); + } + if (metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) { + trackTimeSeriesRoutingHash = true; + } + } + this.createTsidDuringRouting = createTsidDuringRouting; + this.trackTimeSeriesRoutingHash = trackTimeSeriesRoutingHash; addIdWithRoutingHash = indexMode == IndexMode.LOGSDB; - List routingPaths = metadata.getRoutingPaths(); - isRoutingPath = Regex.simpleMatcher(routingPaths.toArray(String[]::new)); - this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, Set.copyOf(routingPaths), null, true); + isRoutingPath = Regex.simpleMatcher(includePaths.toArray(String[]::new)); + this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, Set.copyOf(includePaths), null, true); } public boolean matchesField(String fieldName) { @@ -323,8 +353,12 @@ public boolean matchesField(String fieldName) { @Override public void postProcess(IndexRequest indexRequest) { - // Update the request with the routing hash, if needed. + // Update the request with the routing hash and the tsid, if needed. // This needs to happen in post-processing, after the routing hash is calculated. + if (createTsidDuringRouting) { + assert tsid != null; + indexRequest.tsid(tsid); + } if (trackTimeSeriesRoutingHash) { indexRequest.routing(TimeSeriesRoutingHashFieldMapper.encode(hash)); } else if (addIdWithRoutingHash) { @@ -334,59 +368,79 @@ public void postProcess(IndexRequest indexRequest) { } @Override - public int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source) { + public int indexShard( + String id, + @Nullable String routing, + @Nullable BytesRef tsid, + XContentType sourceType, + BytesReference source + ) { assert Transports.assertNotTransportThread("parsing the _source can get slow"); checkNoRouting(routing); - hash = hashSource(sourceType, source).buildHash(IndexRouting.ExtractFromSource::defaultOnEmpty); + if (createTsidDuringRouting) { + if (tsid == null) { + this.tsid = buildTsid(sourceType, source); + } else { + this.tsid = tsid; + } + hash = hash(this.tsid); + } else { + hash = hashRoutingFields(sourceType, source).buildHash(IndexRouting.ExtractFromSource::defaultOnEmpty); + } int shardId = hashToShardId(hash); return (rerouteIfResharding(shardId)); } public String createId(XContentType sourceType, BytesReference source, byte[] suffix) { - return hashSource(sourceType, source).createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); - } - - public String createId(Map flat, byte[] suffix) { - Builder b = builder(); - for (Map.Entry e : flat.entrySet()) { - if (isRoutingPath.test(e.getKey())) { - if (e.getValue() instanceof List listValue) { - for (Object v : listValue) { - b.addHash(e.getKey(), new BytesRef(v.toString())); - } - } else { - b.addHash(e.getKey(), new BytesRef(e.getValue().toString())); - } - } - } - return b.createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); + return hashRoutingFields(sourceType, source).createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); } private static int defaultOnEmpty() { throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); } - public Builder builder() { - return new Builder(); + public RoutingHashBuilder builder() { + return new RoutingHashBuilder(); + } + + private RoutingHashBuilder hashRoutingFields(XContentType sourceType, BytesReference source) { + RoutingHashBuilder b = builder(); + withFilteredParser(sourceType, source, parser -> b.extractObject(null, parser)); + return b; } - private Builder hashSource(XContentType sourceType, BytesReference source) { - Builder b = builder(); + private BytesRef buildTsid(XContentType sourceType, BytesReference source) { + TsidBuilder b = new TsidBuilder(); + withFilteredParser(sourceType, source, parser -> b.add(parser, XContentParserTsidFunnel.get())); + return b.buildTsid(); + } + + private void withFilteredParser( + XContentType sourceType, + BytesReference source, + CheckedConsumer parserConsumer + ) { try (XContentParser parser = XContentHelper.createParserNotCompressed(parserConfig, source, sourceType)) { parser.nextToken(); // Move to first token if (parser.currentToken() == null) { throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); } parser.nextToken(); - b.extractObject(null, parser); + parserConsumer.accept(parser); ensureExpectedToken(null, parser.nextToken(), parser); } catch (IOException | ParsingException e) { throw new IllegalArgumentException("Error extracting routing: " + e.getMessage(), e); } - return b; } - public class Builder { + /** + * Visible for testing + */ + boolean isCreateTsidDuringRouting() { + return createTsidDuringRouting; + } + + public class RoutingHashBuilder { private final List hashes = new ArrayList<>(); public void addMatching(String fieldName, BytesRef string) { @@ -395,6 +449,11 @@ public void addMatching(String fieldName, BytesRef string) { } } + /** + * Only expected to be called for old indices created before + * {@link IndexVersions#TIME_SERIES_ROUTING_HASH_IN_ID} while creating (during ingestion) + * or synthesizing (at query time) the _id field. + */ public String createId(byte[] suffix, IntSupplier onEmpty) { byte[] idBytes = new byte[4 + suffix.length]; ByteUtils.writeIntLE(buildHash(onEmpty), idBytes, 0); @@ -465,6 +524,16 @@ private int buildHash(IntSupplier onEmpty) { } return hash; } + + private record NameAndHash(BytesRef name, int hash, int order) implements Comparable { + @Override + public int compareTo(NameAndHash o) { + int i = name.compareTo(o.name); + if (i != 0) return i; + // ensures array values are in the order as they appear in the source + return Integer.compare(order, o.order); + } + } } private static int hash(BytesRef ref) { @@ -525,15 +594,78 @@ public void collectSearchShards(String routing, IntConsumer consumer) { private String error(String operation) { return operation + " is not supported because the destination index [" + indexName + "] is in " + indexMode.getName() + " mode"; } - } - private record NameAndHash(BytesRef name, int hash, int order) implements Comparable { - @Override - public int compareTo(NameAndHash o) { - int i = name.compareTo(o.name); - if (i != 0) return i; - // ensures array values are in the order as they appear in the source - return Integer.compare(order, o.order); + static class XContentParserTsidFunnel implements TsidBuilder.ThrowingTsidFunnel { + + private static final XContentParserTsidFunnel INSTANCE = new XContentParserTsidFunnel(); + + static XContentParserTsidFunnel get() { + return INSTANCE; + } + + @Override + public void add(XContentParser value, TsidBuilder tsidBuilder) throws IOException { + extractObject(tsidBuilder, null, value); + } + + private void extractObject(TsidBuilder tsidBuilder, @Nullable String path, XContentParser source) throws IOException { + while (source.currentToken() != Token.END_OBJECT) { + ensureExpectedToken(Token.FIELD_NAME, source.currentToken(), source); + String fieldName = source.currentName(); + String subPath = path == null ? fieldName : path + "." + fieldName; + source.nextToken(); + extractItem(tsidBuilder, subPath, source); + } + } + + private void extractArray(TsidBuilder tsidBuilder, @Nullable String path, XContentParser source) throws IOException { + while (source.currentToken() != Token.END_ARRAY) { + expectValueToken(source.currentToken(), source); + extractItem(tsidBuilder, path, source); + } + } + + private void extractItem(TsidBuilder tsidBuilder, String path, XContentParser source) throws IOException { + switch (source.currentToken()) { + case START_OBJECT: + source.nextToken(); + extractObject(tsidBuilder, path, source); + source.nextToken(); + break; + case VALUE_NUMBER: + switch (source.numberType()) { + case INT -> tsidBuilder.addIntDimension(path, source.intValue()); + case LONG -> tsidBuilder.addLongDimension(path, source.longValue()); + case FLOAT -> tsidBuilder.addDoubleDimension(path, source.floatValue()); + case DOUBLE -> tsidBuilder.addDoubleDimension(path, source.doubleValue()); + case BIG_DECIMAL, BIG_INTEGER -> tsidBuilder.addStringDimension(path, source.text()); + } + source.nextToken(); + break; + case VALUE_BOOLEAN: + tsidBuilder.addBooleanDimension(path, source.booleanValue()); + source.nextToken(); + break; + case VALUE_STRING: + tsidBuilder.addStringDimension(path, source.text()); + source.nextToken(); + break; + case START_ARRAY: + source.nextToken(); + extractArray(tsidBuilder, path, source); + source.nextToken(); + break; + case VALUE_NULL: + source.nextToken(); + break; + default: + throw new ParsingException( + source.getTokenLocation(), + "Cannot extract dimension due to unexpected token [{}]", + source.currentToken() + ); + } + } } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java b/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java new file mode 100644 index 0000000000000..9fe231f60c452 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java @@ -0,0 +1,192 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.cluster.routing; + +import com.dynatrace.hash4j.hashing.HashStream128; +import com.dynatrace.hash4j.hashing.HashStream32; +import com.dynatrace.hash4j.hashing.HashValue128; +import com.dynatrace.hash4j.hashing.Hasher128; +import com.dynatrace.hash4j.hashing.Hasher32; +import com.dynatrace.hash4j.hashing.Hashing; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.index.mapper.RoutingPathFields; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class TsidBuilder { + + private static final int MAX_TSID_VALUE_FIELDS = 16; + private static final Hasher128 HASHER_128 = Hashing.murmur3_128(); + private static final Hasher32 HASHER_32 = Hashing.murmur3_32(); + private final HashStream128 hashStream = HASHER_128.hashStream(); + + private final List dimensions = new ArrayList<>(); + + public void addIntDimension(String path, int value) { + addDimension(path, new HashValue128(1, value)); + } + + public void addLongDimension(String path, long value) { + addDimension(path, new HashValue128(2, value)); + } + + public void addDoubleDimension(String path, double value) { + addDimension(path, new HashValue128(2, Double.doubleToLongBits(value))); + } + + public void addBooleanDimension(String path, boolean value) { + addDimension(path, new HashValue128(4, value ? 1 : 0)); + } + + public void addStringDimension(String path, String value) { + hashStream.reset(); + hashStream.putString(value); + HashValue128 valueHash = hashStream.get(); + addDimension(path, valueHash); + } + + public void addBytesDimension(String path, byte[] value) { + hashStream.reset(); + hashStream.putBytes(value); + HashValue128 valueHash = hashStream.get(); + addDimension(path, valueHash); + } + + public void add(T value, TsidFunnel funnel) { + funnel.add(value, this); + } + + public void add(T value, ThrowingTsidFunnel funnel) throws E { + funnel.add(value, this); + } + + private void addDimension(String path, HashValue128 valueHash) { + hashStream.reset(); + hashStream.putString(path); + HashValue128 pathHash = hashStream.get(); + dimensions.add(new Dimension(path, pathHash, valueHash, dimensions.size())); + } + + public void addAll(TsidBuilder other) { + if (other == null || other.dimensions.isEmpty()) { + return; + } + dimensions.addAll(other.dimensions); + } + + public HashValue128 hash() { + if (dimensions.isEmpty()) { + throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); + } + Collections.sort(dimensions); + HashStream128 hashStream = HASHER_128.hashStream(); + for (Dimension dim : dimensions) { + hashStream.putLong(dim.pathHash.getMostSignificantBits()); + hashStream.putLong(dim.pathHash.getLeastSignificantBits()); + hashStream.putLong(dim.value.getMostSignificantBits()); + hashStream.putLong(dim.value.getLeastSignificantBits()); + } + return hashStream.get(); + } + + /** + * Builds a time series identifier (TSID) based on the dimensions added to this builder. + * This is a slight adaptation of {@link RoutingPathFields#buildHash()} but creates shorter tsids. + * The TSID is a hash that includes: + *

    + *
  • + * A hash of the dimension field names (4 bytes). + * This is to cluster time series that are using the same dimensions together, which makes the encodings more effective. + *
    • + *
  • + * A hash of the dimension field values (1 byte each, up to a maximum of 16 fields). + * This is to cluster time series with similar values together, also helping with making encodings more effective. + *
    • + *
  • + * A hash of all names and values combined (16 bytes). + * This is to avoid hash collisions. + *
    • + *
+ * + * @return a BytesRef containing the TSID + * @throws IllegalArgumentException if no dimensions have been added + */ + public BytesRef buildTsid() { + if (dimensions.isEmpty()) { + throw new IllegalArgumentException("Error extracting dimensions: source didn't contain any routing fields"); + } + + int numberOfValues = Math.min(MAX_TSID_VALUE_FIELDS, dimensions.size()); + byte[] hash = new byte[4 + numberOfValues + 16]; + int index = 0; + + Collections.sort(dimensions); + HashStream32 fieldNameHash = HASHER_32.hashStream(); + HashStream128 dimensionsHash = HASHER_128.hashStream(); + for (int i = 0; i < dimensions.size(); i++) { + Dimension dim = dimensions.get(i); + fieldNameHash.putLong(dim.pathHash.getMostSignificantBits()); + fieldNameHash.putLong(dim.pathHash.getLeastSignificantBits()); + dimensionsHash.putLong(dim.pathHash.getMostSignificantBits()); + dimensionsHash.putLong(dim.pathHash.getLeastSignificantBits()); + dimensionsHash.putLong(dim.value.getMostSignificantBits()); + dimensionsHash.putLong(dim.value.getLeastSignificantBits()); + } + ByteUtils.writeIntLE(fieldNameHash.getAsInt(), hash, index); + index += 4; + + // similarity hash for values + String previousPath = null; + for (int i = 0; i < numberOfValues; i++) { + Dimension dim = dimensions.get(i); + String path = dim.path(); + if (path == previousPath) { + // only add the first value for array fields + continue; + } + hash[index++] = (byte) dim.value().getAsInt(); + previousPath = path; + } + + index = writeHash128(dimensionsHash.get(), hash, index); + return new BytesRef(hash, 0, index); + } + + + private static int writeHash128(HashValue128 hash128, byte[] buffer, int index) { + ByteUtils.writeLongLE(hash128.getLeastSignificantBits(), buffer, index); + index += 8; + ByteUtils.writeLongLE(hash128.getMostSignificantBits(), buffer, index); + index += 8; + return index; + } + + public interface TsidFunnel { + void add(T value, TsidBuilder tsidBuilder); + } + + public interface ThrowingTsidFunnel { + void add(T value, TsidBuilder tsidBuilder) throws E; + } + + private record Dimension(String path, HashValue128 pathHash, HashValue128 value, int order) implements Comparable { + @Override + public int compareTo(Dimension o) { + int i = path.compareTo(o.path); + if (i != 0) return i; + // ensures array values are in the order as they appear in the source + return Integer.compare(order, o.order); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 9f4c5b80ccf23..4010a8025100a 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -225,6 +225,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // TSDB index settings IndexSettings.MODE, IndexMetadata.INDEX_ROUTING_PATH, + IndexMetadata.INDEX_DIMENSIONS, IndexSettings.TIME_SERIES_START_TIME, IndexSettings.TIME_SERIES_END_TIME, IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 06b10bef8514d..d440d919adaef 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.RoutingFields; import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; @@ -113,7 +114,7 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { + public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { return RoutingFields.Noop.INSTANCE; } @@ -146,13 +147,17 @@ void validateWithOtherSettings(Map, Object> settings) { throw new IllegalArgumentException(error(unsupported)); } } - checkSetting(settings, IndexMetadata.INDEX_ROUTING_PATH); + Setting> routingPath = IndexMetadata.INDEX_ROUTING_PATH; + if (isEmpty(settings, routingPath) && isEmpty(settings, IndexMetadata.INDEX_DIMENSIONS)) { + // index.dimensions is a private setting that only gets populated for data streams. + // We don't include it in the error message to not confuse users that are manually creating time series indices + // which is the only case where this error can occur. + throw new IllegalArgumentException(tsdbMode() + " requires a non-empty [" + routingPath.getKey() + "]"); + } } - private static void checkSetting(Map, Object> settings, Setting setting) { - if (Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting))) { - throw new IllegalArgumentException(tsdbMode() + " requires a non-empty [" + setting.getKey() + "]"); - } + private static boolean isEmpty(Map, Object> settings, Setting> setting) { + return Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting)); } private static String error(Setting unsupported) { @@ -213,7 +218,11 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { + public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { + if (source.tsid() != null) { + // If the source already has a _tsid field, we don't need to extract routing from the source. + return RoutingFields.Noop.INSTANCE; + } IndexRouting.ExtractFromSource routing = (IndexRouting.ExtractFromSource) settings.getIndexRouting(); return new RoutingPathFields(routing.builder()); } @@ -295,7 +304,7 @@ public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { + public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { return RoutingFields.Noop.INSTANCE; } @@ -376,7 +385,7 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { + public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { return RoutingFields.Noop.INSTANCE; } @@ -460,6 +469,7 @@ private static CompressedXContent createDefaultMapping(boolean includeHostName) IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING, IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING, IndexMetadata.INDEX_ROUTING_PATH, + IndexMetadata.INDEX_DIMENSIONS, IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS, IndexSettings.TIME_SERIES_START_TIME, IndexSettings.TIME_SERIES_END_TIME @@ -536,7 +546,7 @@ public String getName() { /** * How {@code time_series_dimension} fields are handled by indices in this mode. */ - public abstract RoutingFields buildRoutingFields(IndexSettings settings); + public abstract RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source); /** * @return Whether timestamps should be validated for being withing the time range of an index. diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 2e464afa72b76..a3addab192e68 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -180,6 +180,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT = def(9_031_0_00, Version.LUCENE_10_2_2); public static final IndexVersion DEFAULT_DENSE_VECTOR_TO_BBQ_HNSW = def(9_032_0_00, Version.LUCENE_10_2_2); public static final IndexVersion MATCH_ONLY_TEXT_STORED_AS_BYTES = def(9_033_0_00, Version.LUCENE_10_2_2); + public static final IndexVersion TSID_CREATED_DURING_ROUTING = def(9_034_0_00, Version.LUCENE_10_2_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 15e7ff88350b6..d5eaa5c3136d1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; @@ -93,7 +94,7 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL ) ) ) { - context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser); + context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser, source.tsid()); validateStart(context.parser()); MetadataFieldMapper[] metadataFieldsMappers = mappingLookup.getMapping().getSortedMetadataMappers(); internalParseDocument(metadataFieldsMappers, context); @@ -1078,12 +1079,14 @@ private static class RootDocumentParserContext extends DocumentParserContext { private final long maxAllowedNumNestedDocs; private long numNestedDocs; private boolean docsReversed = false; + private final BytesRef tsid; RootDocumentParserContext( MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse source, - XContentParser parser + XContentParser parser, + BytesRef tsid ) throws IOException { super( mappingLookup, @@ -1092,6 +1095,7 @@ private static class RootDocumentParserContext extends DocumentParserContext { mappingLookup.getMapping().getRoot(), ObjectMapper.Dynamic.getRootDynamic(mappingLookup) ); + this.tsid = tsid; if (mappingLookup.getMapping().getRoot().subobjects() == ObjectMapper.Subobjects.ENABLED) { this.parser = DotExpandingXContentParser.expandDots(parser, this.path); } else { @@ -1149,6 +1153,11 @@ protected void addDoc(LuceneDocument doc) { this.documents.add(doc); } + @Override + public BytesRef getTsid() { + return this.tsid; + } + @Override public Iterable nonRootDocuments() { if (docsReversed) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index b77c0426c23d4..8c7f44df3f70d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -14,6 +14,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; @@ -116,6 +117,11 @@ public XContentParser.Token getImmediateXContentParent() { public boolean isImmediateParentAnArray() { return in.isImmediateParentAnArray(); } + + @Override + public BytesRef getTsid() { + return in.getTsid(); + } } /** @@ -265,7 +271,7 @@ protected DocumentParserContext( null, null, SeqNoFieldMapper.SequenceIDFields.emptySeqID(mappingParserContext.getIndexSettings().seqNoIndexOptions()), - RoutingFields.fromIndexSettings(mappingParserContext.getIndexSettings()), + RoutingFields.fromIndexSettings(mappingParserContext.getIndexSettings(), source), parent, dynamic, new HashSet<>(), @@ -871,6 +877,9 @@ public final MapperBuilderContext createDynamicMapperBuilderContext() { protected abstract void addDoc(LuceneDocument doc); + @Nullable + public abstract BytesRef getTsid(); + /** * Find a dynamic mapping template for the given field and its matching type * diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index 741252f98473b..12228760faf35 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -67,9 +67,10 @@ final class TsIdLoader implements IdLoader { } public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException { - IndexRouting.ExtractFromSource.Builder[] builders = null; + IndexRouting.ExtractFromSource.RoutingHashBuilder[] builders = null; if (indexRouting != null) { - builders = new IndexRouting.ExtractFromSource.Builder[docIdsInLeaf.length]; + // this branch is for legacy indices before IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID + builders = new IndexRouting.ExtractFromSource.RoutingHashBuilder[docIdsInLeaf.length]; for (int i = 0; i < builders.length; i++) { builders[i] = indexRouting.builder(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java index 4d8d8fdcbd296..0ad4f4e0b49aa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java @@ -22,8 +22,8 @@ public interface RoutingFields { /** * Collect routing fields from index settings */ - static RoutingFields fromIndexSettings(IndexSettings indexSettings) { - return indexSettings.getMode().buildRoutingFields(indexSettings); + static RoutingFields fromIndexSettings(IndexSettings indexSettings, SourceToParse source) { + return indexSettings.getMode().buildRoutingFields(indexSettings, source); } /** @@ -45,6 +45,8 @@ default RoutingFields addString(String fieldName, String value) { RoutingFields addBoolean(String fieldName, boolean value); + boolean isNoop(); + /** * Noop implementation that doesn't perform validations on routing fields */ @@ -81,5 +83,10 @@ public RoutingFields addUnsignedLong(String fieldName, long value) { public RoutingFields addBoolean(String fieldName, boolean value) { return this; } + + @Override + public boolean isNoop() { + return true; + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java index 73baca1bf3fdb..361163ea1be66 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java @@ -59,9 +59,9 @@ public final class RoutingPathFields implements RoutingFields { * Builds the routing. Used for building {@code _id}. If null then skipped. */ @Nullable - private final IndexRouting.ExtractFromSource.Builder routingBuilder; + private final IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder; - public RoutingPathFields(@Nullable IndexRouting.ExtractFromSource.Builder routingBuilder) { + public RoutingPathFields(@Nullable IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder) { this.routingBuilder = routingBuilder; } @@ -69,7 +69,7 @@ SortedMap> routingValues() { return Collections.unmodifiableSortedMap(routingValues); } - IndexRouting.ExtractFromSource.Builder routingBuilder() { + IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder() { return routingBuilder; } @@ -207,6 +207,11 @@ public RoutingFields addBoolean(String fieldName, boolean value) { return this; } + @Override + public boolean isNoop() { + return false; + } + private void add(String fieldName, BytesReference encoded) throws IOException { BytesRef name = new BytesRef(fieldName); List values = routingValues.get(name); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java index 5396fdef0f041..8a2ecb126c6cb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.Nullable; @@ -24,6 +25,8 @@ public class SourceToParse { private final String id; + private final @Nullable BytesRef tsid; + private final @Nullable String routing; private final XContentType xContentType; @@ -41,7 +44,8 @@ public SourceToParse( @Nullable String routing, Map dynamicTemplates, boolean includeSourceOnError, - XContentMeteringParserDecorator meteringParserDecorator + XContentMeteringParserDecorator meteringParserDecorator, + @Nullable BytesRef tsid ) { this.id = id; // we always convert back to byte array, since we store it and Field only supports bytes.. @@ -52,14 +56,15 @@ public SourceToParse( this.dynamicTemplates = Objects.requireNonNull(dynamicTemplates); this.includeSourceOnError = includeSourceOnError; this.meteringParserDecorator = meteringParserDecorator; + this.tsid = tsid; } public SourceToParse(String id, BytesReference source, XContentType xContentType) { - this(id, source, xContentType, null, Map.of(), true, XContentMeteringParserDecorator.NOOP); + this(id, source, xContentType, null, Map.of(), true, XContentMeteringParserDecorator.NOOP, null); } public SourceToParse(String id, BytesReference source, XContentType xContentType, String routing) { - this(id, source, xContentType, routing, Map.of(), true, XContentMeteringParserDecorator.NOOP); + this(id, source, xContentType, routing, Map.of(), true, XContentMeteringParserDecorator.NOOP, null); } public SourceToParse( @@ -67,9 +72,10 @@ public SourceToParse( BytesReference source, XContentType xContentType, String routing, - Map dynamicTemplates + Map dynamicTemplates, + BytesRef tsid ) { - this(id, source, xContentType, routing, dynamicTemplates, true, XContentMeteringParserDecorator.NOOP); + this(id, source, xContentType, routing, dynamicTemplates, true, XContentMeteringParserDecorator.NOOP, tsid); } public BytesReference source() { @@ -113,4 +119,8 @@ public XContentMeteringParserDecorator getMeteringParserDecorator() { public boolean getIncludeSourceOnError() { return includeSourceOnError; } + + public BytesRef tsid() { + return tsid; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index b94fa64b42428..0f3776f1db9bb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -14,6 +14,7 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -150,16 +151,21 @@ private TimeSeriesIdFieldMapper(boolean useDocValuesSkipper) { public void postParse(DocumentParserContext context) throws IOException { assert fieldType().isIndexed() == false; - final RoutingPathFields routingPathFields = (RoutingPathFields) context.getRoutingFields(); final BytesRef timeSeriesId; + final RoutingPathFields routingPathFields; if (getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ID_HASHING)) { + routingPathFields = (RoutingPathFields) context.getRoutingFields(); long limit = context.indexSettings().getValue(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); int size = routingPathFields.routingValues().size(); if (size > limit) { throw new MapperException("Too many dimension fields [" + size + "], max [" + limit + "] dimension fields allowed"); } timeSeriesId = buildLegacyTsid(routingPathFields).toBytesRef(); + } else if (context.getTsid() != null) { + routingPathFields = null; + timeSeriesId = context.getTsid(); } else { + routingPathFields = (RoutingPathFields) context.getRoutingFields(); timeSeriesId = routingPathFields.buildHash().toBytesRef(); } @@ -169,13 +175,15 @@ public void postParse(DocumentParserContext context) throws IOException { context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); } - BytesRef uidEncoded = TsidExtractingIdFieldMapper.createField( - context, - getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID) - ? routingPathFields.routingBuilder() - : null, - timeSeriesId - ); + IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder; + if (getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID) && routingPathFields != null) { + // For legacy indices, we need to create the routing hash from the routing path fields. + routingBuilder = routingPathFields.routingBuilder(); + } else { + // For newer indices, the routing hash is stored in SourceToParse#routing, so we can use that directly. + routingBuilder = null; + } + BytesRef uidEncoded = TsidExtractingIdFieldMapper.createField(context, routingBuilder, timeSeriesId); // We need to add the uid or id to nested Lucene documents so that when a document gets deleted, the nested documents are // also deleted. Usually this happens when the nested document is created (in DocumentParserContext#createNestedContext), but diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 8aca004949209..b48b22520ea36 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -48,7 +48,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext public static BytesRef createField( DocumentParserContext context, - IndexRouting.ExtractFromSource.Builder routingBuilder, + IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder, BytesRef tsid ) { final long timestamp = DataStreamTimestampFieldMapper.extractTimestampValue(context.doc()); @@ -115,7 +115,7 @@ public static String createId(int routingHash, BytesRef tsid, long timestamp) { public static String createId( boolean dynamicMappersExists, - IndexRouting.ExtractFromSource.Builder routingBuilder, + IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder, BytesRef tsid, long timestamp, byte[] suffix diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java index 93ef04ddd159a..953ac8f7a6941 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java @@ -177,7 +177,7 @@ private void addField(Context context, ContentPath path, String currentName, Str fields.add(new SortedSetDocValuesField(rootFieldFullPath, bytesValue)); fields.add(new SortedSetDocValuesField(keyedFieldFullPath, bytesKeyedValue)); - if (fieldType.isDimension() == false) { + if (fieldType.isDimension() == false || context.documentParserContext().getRoutingFields().isNoop()) { return; } diff --git a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java index 9f5c9db1452d9..68900c612f513 100644 --- a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java @@ -8,6 +8,7 @@ */ package org.elasticsearch.action.index; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocWriteRequest; @@ -480,6 +481,7 @@ public void testSerialization() throws IOException { assertThat(copy.getFinalPipeline(), equalTo(indexRequest.getFinalPipeline())); assertThat(copy.ifPrimaryTerm(), equalTo(indexRequest.ifPrimaryTerm())); assertThat(copy.isRequireDataStream(), equalTo(indexRequest.isRequireDataStream())); + assertThat(copy.tsid(), equalTo(indexRequest.tsid())); } private IndexRequest createTestInstance() { @@ -495,6 +497,7 @@ private IndexRequest createTestInstance() { for (int i = 0; i < randomIntBetween(0, 20); i++) { indexRequest.addPipeline(randomAlphaOfLength(20)); } + indexRequest.tsid(randomFrom((BytesRef) null, new BytesRef(randomAlphaOfLength(20)))); return indexRequest; } } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 1db6192eee80e..4f65835e69d4d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; @@ -460,7 +461,7 @@ public void testRequiredRouting() { */ private int shardIdFromSimple(IndexRouting indexRouting, String id, @Nullable String routing) { return switch (between(0, 3)) { - case 0 -> indexRouting.indexShard(id, routing, null, null); + case 0 -> indexRouting.indexShard(id, routing, null, null, null); case 1 -> indexRouting.updateShard(id, routing); case 2 -> indexRouting.deleteShard(id, routing); case 3 -> indexRouting.getShard(id, routing); @@ -493,7 +494,7 @@ public void testRoutingPathEmptySource() throws IOException { IndexRouting routing = indexRoutingForPath(between(1, 5), randomAlphaOfLength(5)); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(Map.of())) + () -> routing.indexShard(randomAlphaOfLength(5), null, null, XContentType.JSON, source(Map.of())) ); assertThat(e.getMessage(), equalTo("Error extracting routing: source didn't contain any routing fields")); } @@ -502,7 +503,7 @@ public void testRoutingPathMismatchSource() throws IOException { IndexRouting routing = indexRoutingForPath(between(1, 5), "foo"); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(Map.of("bar", "dog"))) + () -> routing.indexShard(randomAlphaOfLength(5), null, null, XContentType.JSON, source(Map.of("bar", "dog"))) ); assertThat(e.getMessage(), equalTo("Error extracting routing: source didn't contain any routing fields")); } @@ -523,7 +524,7 @@ public void testRoutingIndexWithRouting() throws IOException { String docRouting = randomAlphaOfLength(5); Exception e = expectThrows( IllegalArgumentException.class, - () -> indexRouting.indexShard(randomAlphaOfLength(5), docRouting, XContentType.JSON, source) + () -> indexRouting.indexShard(randomAlphaOfLength(5), docRouting, null, XContentType.JSON, source) ); assertThat( e.getMessage(), @@ -543,7 +544,7 @@ public void testRoutingPathCollectSearchWithRouting() throws IOException { public void testRoutingPathOneTopLevel() throws IOException { int shards = between(2, 1000); IndexRouting routing = indexRoutingForPath(shards, "foo"); - assertIndexShard(routing, Map.of("foo", "cat", "bar", "dog"), Math.floorMod(hash(List.of("foo", "cat")), shards)); + assertIndexShard(routing, Map.of("foo", "cat", "bar", "dog"), List.of("foo", "cat"), shards); } public void testRoutingPathManyTopLevel() throws IOException { @@ -552,7 +553,8 @@ public void testRoutingPathManyTopLevel() throws IOException { assertIndexShard( routing, Map.of("foo", "cat", "bar", "dog", "foa", "a", "fob", "b"), - Math.floorMod(hash(List.of("foa", "a", "fob", "b", "foo", "cat")), shards) // Note that the fields are sorted + List.of("foa", "a", "fob", "b", "foo", "cat"), + shards // Note that the fields are sorted ); } @@ -562,7 +564,8 @@ public void testRoutingPathOneSub() throws IOException { assertIndexShard( routing, Map.of("foo", Map.of("bar", "cat"), "baz", "dog"), - Math.floorMod(hash(List.of("foo.bar", "cat")), shards) + List.of("foo.bar", "cat"), + shards ); } @@ -572,31 +575,32 @@ public void testRoutingPathManySubs() throws IOException { assertIndexShard( routing, Map.of("foo", Map.of("a", "cat"), "bar", Map.of("thing", "yay", "this", "too")), - Math.floorMod(hash(List.of("bar.thing", "yay", "bar.this", "too", "foo.a", "cat")), shards) + List.of("bar.thing", "yay", "bar.this", "too", "foo.a", "cat"), + shards ); } public void testRoutingPathDotInName() throws IOException { int shards = between(2, 1000); IndexRouting routing = indexRoutingForPath(shards, "foo.bar"); - assertIndexShard(routing, Map.of("foo.bar", "cat", "baz", "dog"), Math.floorMod(hash(List.of("foo.bar", "cat")), shards)); + assertIndexShard(routing, Map.of("foo.bar", "cat", "baz", "dog"), List.of("foo.bar", "cat"), shards); } public void testRoutingPathNumbersInSource() throws IOException { int shards = between(2, 1000); - IndexRouting routing = indexRoutingForPath(shards, "foo"); + IndexRouting routing = indexRoutingForRoutingPath(IndexVersions.MATCH_ONLY_TEXT_STORED_AS_BYTES, shards, "foo"); long randomLong = randomLong(); - assertIndexShard(routing, Map.of("foo", randomLong), Math.floorMod(hash(List.of("foo", Long.toString(randomLong))), shards)); + assertIndexShard(routing, Map.of("foo", randomLong), List.of("foo", Long.toString(randomLong)), shards); double randomDouble = randomDouble(); - assertIndexShard(routing, Map.of("foo", randomDouble), Math.floorMod(hash(List.of("foo", Double.toString(randomDouble))), shards)); - assertIndexShard(routing, Map.of("foo", 123), Math.floorMod(hash(List.of("foo", "123")), shards)); + assertIndexShard(routing, Map.of("foo", randomDouble), List.of("foo", Double.toString(randomDouble)), shards); + assertIndexShard(routing, Map.of("foo", 123), List.of("foo", 123), shards); } public void testRoutingPathBooleansInSource() throws IOException { int shards = between(2, 1000); IndexRouting routing = indexRoutingForPath(shards, "foo"); - assertIndexShard(routing, Map.of("foo", true), Math.floorMod(hash(List.of("foo", "true")), shards)); - assertIndexShard(routing, Map.of("foo", false), Math.floorMod(hash(List.of("foo", "false")), shards)); + assertIndexShard(routing, Map.of("foo", true), List.of("foo", "true"), shards); + assertIndexShard(routing, Map.of("foo", false), List.of("foo", "false"), shards); } public void testRoutingPathArraysInSource() throws IOException { @@ -606,7 +610,8 @@ public void testRoutingPathArraysInSource() throws IOException { routing, Map.of("c", List.of(true), "d", List.of(), "a", List.of("foo", "bar", "foo"), "b", List.of(21, 42)), // Note that the fields are sorted - Math.floorMod(hash(List.of("a", "foo", "a", "bar", "a", "foo", "b", "21", "b", "42", "c", "true")), shards) + List.of("a", "foo", "a", "bar", "a", "foo", "b", 21, "b", 42, "c", true), + shards ); } @@ -617,7 +622,7 @@ public void testRoutingPathObjectArraysInSource() throws IOException { BytesReference source = source(Map.of("a", List.of("foo", Map.of("foo", "bar")))); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source) + () -> routing.indexShard(randomAlphaOfLength(5), null, null, XContentType.JSON, source) ); assertThat( e.getMessage(), @@ -626,10 +631,10 @@ public void testRoutingPathObjectArraysInSource() throws IOException { } public void testRoutingPathBwc() throws IOException { - IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random()); - IndexRouting routing = indexRoutingForPath(version, 8, "dim.*,other.*,top"); + IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.TSID_CREATED_DURING_ROUTING); + IndexRouting routing = indexRoutingForRoutingPath(version, 8, "dim.*,other.*,top"); /* - * These when we first added routing_path. If these values change + * These are the expected shards when we first added routing_path. If these values change * time series will be routed to unexpected shards. You may modify * them with a new index created version, but when you do you must * copy this test and patch the versions at the top. Because newer @@ -645,6 +650,29 @@ public void testRoutingPathBwc() throws IOException { assertIndexShard(routing, Map.of("dim.a", "a"), 4); } + public void testRoutingPathBwcAfterTsidBasedRouting() throws IOException { + IndexRouting routing = indexRoutingForDimensions(IndexVersion.current(), 8, "dim.*,other.*,top"); + /* + * These are the expected shards after tsid based routing. If these values change + * time series will be routed to unexpected shards. You may modify + * them with a new index created version, but when you do you must + * copy this test and patch the versions at the top. Because newer + * versions of Elasticsearch must continue to route based on the + * version on the index. + */ + assertIndexShard(routing, Map.of("dim", Map.of("a", "a")), 5); + assertIndexShard(routing, Map.of("dim", Map.of("a", "b")), 1); + assertIndexShard(routing, Map.of("dim", Map.of("c", "d")), 6); + assertIndexShard(routing, Map.of("other", Map.of("a", "a")), 1); + assertIndexShard(routing, Map.of("top", "a"), 2); + assertIndexShard(routing, Map.of("dim", Map.of("c", "d"), "top", "b"), 2); + assertIndexShard(routing, Map.of("dim.a", "a"), 5); + assertIndexShard(routing, Map.of("dim.a", 1), 7); + assertIndexShard(routing, Map.of("dim.a", "1"), 5); + assertIndexShard(routing, Map.of("dim.a", true), 3); + assertIndexShard(routing, Map.of("dim.a", "true"), 4); + } + public void testRoutingPathReadWithInvalidString() throws IOException { int shards = between(2, 1000); IndexRouting indexRouting = indexRoutingForPath(shards, "foo"); @@ -678,9 +706,9 @@ public void testRoutingPathLogsdb() throws IOException { assertNull(req.id()); // Verify that routing uses the field name and value in the routing path. - int expectedShard = Math.floorMod(hash(List.of("foo", "A")), shards); + int expectedShard = expectedShard(routing, List.of("foo", "A"), shards); BytesReference sourceBytes = source(Map.of("foo", "A", "bar", "B")); - assertEquals(expectedShard, routing.indexShard(null, null, XContentType.JSON, sourceBytes)); + assertEquals(expectedShard, routing.indexShard(null, null, null, XContentType.JSON, sourceBytes)); // Verify that the request id gets updated to contain the routing hash. routing.postProcess(req); @@ -697,10 +725,14 @@ private int shardIdForReadFromSourceExtracting(IndexRouting indexRouting, String } private IndexRouting indexRoutingForPath(int shards, String path) { - return indexRoutingForPath(IndexVersion.current(), shards, path); + return randomBoolean() + // old way of routing paths created during routing + ? indexRoutingForRoutingPath(IndexVersion.current(), shards, path) + // current way of routing paths created during routing via tsid + : indexRoutingForDimensions(IndexVersion.current(), shards, path); } - private IndexRouting indexRoutingForPath(IndexVersion createdVersion, int shards, String path) { + private IndexRouting indexRoutingForRoutingPath(IndexVersion createdVersion, int shards, String path) { return IndexRouting.fromIndexMetadata( IndexMetadata.builder("test") .settings( @@ -714,18 +746,41 @@ private IndexRouting indexRoutingForPath(IndexVersion createdVersion, int shards ); } + private IndexRouting indexRoutingForDimensions(IndexVersion createdVersion, int shards, String path) { + return IndexRouting.fromIndexMetadata( + IndexMetadata.builder("test") + .settings( + settings(createdVersion).put(IndexMetadata.INDEX_DIMENSIONS.getKey(), path) + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .build() + ) + .numberOfShards(shards) + .numberOfReplicas(1) + .build() + ); + } + + private void assertIndexShard(IndexRouting routing, Map source, List keysAndValues, int shards) + throws IOException { + assertIndexShard(routing, source, expectedShard(routing, keysAndValues, shards)); + } + private void assertIndexShard(IndexRouting routing, Map source, int expectedShard) throws IOException { byte[] suffix = randomSuffix(); BytesReference sourceBytes = source(source); - assertThat(routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, sourceBytes), equalTo(expectedShard)); + assertThat(routing.indexShard(randomAlphaOfLength(5), null, null, XContentType.JSON, sourceBytes), equalTo(expectedShard)); IndexRouting.ExtractFromSource r = (IndexRouting.ExtractFromSource) routing; + if (r.isCreateTsidDuringRouting()) { + // The rest of the assertions are only relevant when only the routing hash is created + return; + } String idFromSource = r.createId(XContentType.JSON, sourceBytes, suffix); assertThat(shardIdForReadFromSourceExtracting(routing, idFromSource), equalTo(expectedShard)); Map flattened = flatten(source); - String idFromFlattened = r.createId(flattened, suffix); + String idFromFlattened = r.createId(XContentType.JSON, sourceBytes, suffix); assertThat(idFromFlattened, equalTo(idFromSource)); - IndexRouting.ExtractFromSource.Builder b = r.builder(); + IndexRouting.ExtractFromSource.RoutingHashBuilder b = r.builder(); for (Map.Entry e : flattened.entrySet()) { if (e.getValue() instanceof List listValue) { listValue.forEach(v -> b.addMatching(e.getKey(), new BytesRef(v.toString()))); @@ -733,7 +788,7 @@ private void assertIndexShard(IndexRouting routing, Map source, b.addMatching(e.getKey(), new BytesRef(e.getValue().toString())); } } - String idFromBuilder = b.createId(suffix, () -> { throw new AssertionError(); }); + String idFromBuilder = b.createId(suffix, () -> {throw new AssertionError();}); assertThat(idFromBuilder, equalTo(idFromSource)); } @@ -776,17 +831,46 @@ private void flatten(Map result, String path, Map m) { } } + private int expectedShard(IndexRouting routing, List keysAndValues, int shards) { + return Math.floorMod(hash(routing, keysAndValues), shards); + } + /** * Build the hash we expect from the extracter. */ - private int hash(List keysAndValues) { + private int hash(IndexRouting routing, List keysAndValues) { + if (routing instanceof IndexRouting.ExtractFromSource extractFromSource && extractFromSource.isCreateTsidDuringRouting()) { + return tsidBasedRoutingHash(keysAndValues); + } + return legacyRoutingHash(keysAndValues); + } + + private int legacyRoutingHash(List keysAndValues) { assertThat(keysAndValues.size() % 2, equalTo(0)); int hash = 0; for (int i = 0; i < keysAndValues.size(); i += 2) { - int keyHash = StringHelper.murmurhash3_x86_32(new BytesRef(keysAndValues.get(i)), 0); - int valueHash = StringHelper.murmurhash3_x86_32(new BytesRef(keysAndValues.get(i + 1)), 0); + int keyHash = StringHelper.murmurhash3_x86_32(new BytesRef(keysAndValues.get(i).toString()), 0); + int valueHash = StringHelper.murmurhash3_x86_32(new BytesRef(keysAndValues.get(i + 1).toString()), 0); hash = hash * 31 + (keyHash ^ valueHash); } return hash; } + + private static int tsidBasedRoutingHash(List keysAndValues) { + TsidBuilder tsidBuilder = new TsidBuilder(); + for (int i = 0; i < keysAndValues.size(); i += 2) { + String key = keysAndValues.get(i).toString(); + Object value = keysAndValues.get(i + 1); + if (value instanceof String sValue) { + tsidBuilder.addStringDimension(key, sValue); + } else if (value instanceof Boolean bValue) { + tsidBuilder.addBooleanDimension(key, bValue); + } else if (value instanceof Integer iValue) { + tsidBuilder.addIntDimension(key, iValue); + } else { + throw new IllegalArgumentException("Unsupported value type for TSID routing: " + value.getClass()); + } + } + return StringHelper.murmurhash3_x86_32(tsidBuilder.buildTsid(), 0); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index c7055c9ff438a..5f00769bf1848 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -667,7 +667,7 @@ public void testTemplateWithoutMatchPredicates() throws Exception { {"foo": "41.12,-71.34", "bar": "41.12,-71.34"} """; ParsedDocument doc = mapperService.documentMapper() - .parse(new SourceToParse("1", new BytesArray(json), XContentType.JSON, null, Map.of("foo", "geo_point"))); + .parse(new SourceToParse("1", new BytesArray(json), XContentType.JSON, null, Map.of("foo", "geo_point"), null)); assertThat(doc.rootDoc().getFields("foo"), hasSize(1)); assertThat(doc.rootDoc().getFields("bar"), hasSize(1)); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java index eeb1a349d8bbc..c75252d1d4e71 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java @@ -67,7 +67,8 @@ private Object setupAndInvokeBlockLoader(MapperService mapperService, XContentBu null, Map.of(), true, - XContentMeteringParserDecorator.NOOP + XContentMeteringParserDecorator.NOOP, + null ); LuceneDocument doc = mapperService.documentMapper().parse(source).rootDoc(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 71452917465a7..75c0b3cf6cba7 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -394,7 +394,7 @@ protected static SourceToParse source( XContentBuilder builder = JsonXContent.contentBuilder().startObject(); build.accept(builder); builder.endObject(); - return new SourceToParse(id, BytesReference.bytes(builder), XContentType.JSON, routing, dynamicTemplates); + return new SourceToParse(id, BytesReference.bytes(builder), XContentType.JSON, routing, dynamicTemplates, null); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index 49fe9d30239ae..799f0da58f827 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; @@ -105,4 +106,9 @@ public LuceneDocument rootDoc() { protected void addDoc(LuceneDocument doc) { throw new UnsupportedOperationException(); } + + @Override + public BytesRef getTsid() { + return null; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 77d7bc5d5f7c5..152e8ba16f7ad 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -2423,7 +2423,7 @@ synchronized String routingKeyForShard(Index index, int shard, Random random) { IndexRouting indexRouting = IndexRouting.fromIndexMetadata(clusterState.metadata().getProject().getIndexSafe(index)); while (true) { String routing = RandomStrings.randomAsciiLettersOfLength(random, 10); - if (shard == indexRouting.indexShard("id", routing, null, null)) { + if (shard == indexRouting.indexShard("id", routing, null, null, null)) { return routing; } } From 41d0e28670888fc74ea128c71898dd9c2e54ac8d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 8 Aug 2025 09:17:11 +0000 Subject: [PATCH 02/62] [CI] Auto commit changes from spotless --- .../datastreams/DataStreamIndexSettingsProvider.java | 4 ++-- .../org/elasticsearch/cluster/routing/TsidBuilder.java | 1 - .../elasticsearch/cluster/routing/IndexRoutingTests.java | 9 ++------- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index f69b3e6b90195..9c334adcd1134 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -128,7 +128,7 @@ public Settings getAdditionalIndexSettings( ); if (dimensions.isEmpty() == false) { // TODO handle the case when adding a dimension field to the mappings of an existing index - // at the moment, the index.dimensions setting is only set when an index is created + // at the moment, the index.dimensions setting is only set when an index is created builder.putList(INDEX_DIMENSIONS.getKey(), dimensions); } } @@ -197,7 +197,7 @@ private List findDimensionFields(String indexName, Settings allSettings, } MappingParserContext parserContext = mapperService.parserContext(); - for (Iterator iterator = template.pathMatch().iterator(); iterator.hasNext(); ) { + for (Iterator iterator = template.pathMatch().iterator(); iterator.hasNext();) { var mapper = parserContext.typeParser(mappingSnippetType) .parse(iterator.next(), mappingSnippet, parserContext) .build(MapperBuilderContext.root(false, false)); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java b/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java index 9fe231f60c452..ea74560ff71bb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java @@ -163,7 +163,6 @@ public BytesRef buildTsid() { return new BytesRef(hash, 0, index); } - private static int writeHash128(HashValue128 hash128, byte[] buffer, int index) { ByteUtils.writeLongLE(hash128.getLeastSignificantBits(), buffer, index); index += 8; diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 4f65835e69d4d..204799be2bc37 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -561,12 +561,7 @@ public void testRoutingPathManyTopLevel() throws IOException { public void testRoutingPathOneSub() throws IOException { int shards = between(2, 1000); IndexRouting routing = indexRoutingForPath(shards, "foo.*"); - assertIndexShard( - routing, - Map.of("foo", Map.of("bar", "cat"), "baz", "dog"), - List.of("foo.bar", "cat"), - shards - ); + assertIndexShard(routing, Map.of("foo", Map.of("bar", "cat"), "baz", "dog"), List.of("foo.bar", "cat"), shards); } public void testRoutingPathManySubs() throws IOException { @@ -788,7 +783,7 @@ private void assertIndexShard(IndexRouting routing, Map source, b.addMatching(e.getKey(), new BytesRef(e.getValue().toString())); } } - String idFromBuilder = b.createId(suffix, () -> {throw new AssertionError();}); + String idFromBuilder = b.createId(suffix, () -> { throw new AssertionError(); }); assertThat(idFromBuilder, equalTo(idFromSource)); } From 877de11e0ed4868190b07d4be36439cd581fdae7 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 8 Aug 2025 12:00:06 +0200 Subject: [PATCH 03/62] Add notice file for hash4j --- server/licenses/hash4j-NOTICE.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 server/licenses/hash4j-NOTICE.txt diff --git a/server/licenses/hash4j-NOTICE.txt b/server/licenses/hash4j-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d From c71db0766643b6004646e700da5dd2a5c7050d83 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Thu, 7 Aug 2025 18:23:08 +0200 Subject: [PATCH 04/62] Use optimized text --- .../cluster/routing/IndexRouting.java | 4 ++-- .../cluster/routing/TsidBuilder.java | 19 ++++++++++++++++++- .../cluster/routing/IndexRoutingTests.java | 16 ++++++++-------- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index e256edbb97cc4..4d752b7597dad 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -638,7 +638,7 @@ private void extractItem(TsidBuilder tsidBuilder, String path, XContentParser so case LONG -> tsidBuilder.addLongDimension(path, source.longValue()); case FLOAT -> tsidBuilder.addDoubleDimension(path, source.floatValue()); case DOUBLE -> tsidBuilder.addDoubleDimension(path, source.doubleValue()); - case BIG_DECIMAL, BIG_INTEGER -> tsidBuilder.addStringDimension(path, source.text()); + case BIG_DECIMAL, BIG_INTEGER -> tsidBuilder.addStringDimension(path, source.optimizedText().bytes()); } source.nextToken(); break; @@ -647,7 +647,7 @@ private void extractItem(TsidBuilder tsidBuilder, String path, XContentParser so source.nextToken(); break; case VALUE_STRING: - tsidBuilder.addStringDimension(path, source.text()); + tsidBuilder.addStringDimension(path, source.optimizedText().bytes()); source.nextToken(); break; case START_ARRAY: diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java b/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java index ea74560ff71bb..38bff99917635 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java @@ -19,6 +19,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.index.mapper.RoutingPathFields; +import org.elasticsearch.xcontent.XContentString; import java.util.ArrayList; import java.util.Collections; @@ -50,8 +51,24 @@ public void addBooleanDimension(String path, boolean value) { } public void addStringDimension(String path, String value) { + addStringDimension(path, new BytesRef(value)); + } + + private void addStringDimension(String path, BytesRef value) { + addStringDimension(path, value.bytes, value.offset, value.length); + } + + public void addStringDimension(String path, XContentString.UTF8Bytes value) { + addStringDimension(path, value.bytes(), value.offset(), value.length()); + } + + public void addStringDimension(String path, byte[] value) { + addStringDimension(path, value, 0, value.length); + } + + private void addStringDimension(String path, byte[] bytes, int offset, int length) { hashStream.reset(); - hashStream.putString(value); + hashStream.putBytes(bytes, offset, length); HashValue128 valueHash = hashStream.get(); addDimension(path, valueHash); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 204799be2bc37..9e098a2c31ee6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -655,17 +655,17 @@ public void testRoutingPathBwcAfterTsidBasedRouting() throws IOException { * versions of Elasticsearch must continue to route based on the * version on the index. */ - assertIndexShard(routing, Map.of("dim", Map.of("a", "a")), 5); - assertIndexShard(routing, Map.of("dim", Map.of("a", "b")), 1); - assertIndexShard(routing, Map.of("dim", Map.of("c", "d")), 6); - assertIndexShard(routing, Map.of("other", Map.of("a", "a")), 1); - assertIndexShard(routing, Map.of("top", "a"), 2); - assertIndexShard(routing, Map.of("dim", Map.of("c", "d"), "top", "b"), 2); - assertIndexShard(routing, Map.of("dim.a", "a"), 5); + assertIndexShard(routing, Map.of("dim", Map.of("a", "a")), 6); + assertIndexShard(routing, Map.of("dim", Map.of("a", "b")), 0); + assertIndexShard(routing, Map.of("dim", Map.of("c", "d")), 5); + assertIndexShard(routing, Map.of("other", Map.of("a", "a")), 2); + assertIndexShard(routing, Map.of("top", "a"), 0); + assertIndexShard(routing, Map.of("dim", Map.of("c", "d"), "top", "b"), 3); + assertIndexShard(routing, Map.of("dim.a", "a"), 6); assertIndexShard(routing, Map.of("dim.a", 1), 7); assertIndexShard(routing, Map.of("dim.a", "1"), 5); assertIndexShard(routing, Map.of("dim.a", true), 3); - assertIndexShard(routing, Map.of("dim.a", "true"), 4); + assertIndexShard(routing, Map.of("dim.a", "true"), 6); } public void testRoutingPathReadWithInvalidString() throws IOException { From 0222d93e9cca4a0b174d17fd42a5380a44cf8e4f Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Sat, 9 Aug 2025 15:09:02 +0200 Subject: [PATCH 05/62] Fix DataStreamIndexSettingsProviderTests --- .../DataStreamIndexSettingsProviderTests.java | 47 +++++++++++++------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index b85b3f6e7ae39..318623a81fce2 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -33,7 +33,6 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.FORMATTER; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -70,6 +69,18 @@ public void testGetAdditionalIndexSettings() throws Exception { "field3": { "type": "keyword", "time_series_dimension": true + }, + "field4": { + "type": "long", + "time_series_dimension": true + }, + "field5": { + "type": "ip", + "time_series_dimension": true + }, + "field6": { + "type": "boolean", + "time_series_dimension": true } } } @@ -91,7 +102,7 @@ public void testGetAdditionalIndexSettings() throws Exception { assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("field3")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field3", "field4", "field5", "field6")); } public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throws Exception { @@ -105,7 +116,7 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw "_doc": { "properties": { "field1": { - "type": "keyword" + "type": "keyword", "time_series_dimension": true }, "field2": { @@ -132,10 +143,11 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: // (in production the index.mode setting is usually provided in an index or component template) result = builder().put(result).put("index.mode", "time_series").build(); - assertThat(result.size(), equalTo(3)); + assertThat(result.size(), equalTo(4)); assertThat(result.get(IndexSettings.MODE.getKey()), equalTo("time_series")); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field2", "field3")); } public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception { @@ -206,7 +218,7 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception { assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field1", "field3")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field3")); } public void testGetAdditionalIndexSettingsNoMappings() { @@ -461,7 +473,7 @@ public void testGenerateRoutingPathFromDynamicTemplate() throws Exception { assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); } public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntries() throws Exception { @@ -502,10 +514,10 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); assertThat( - IndexMetadata.INDEX_ROUTING_PATH.get(result), + IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*") ); - List routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result); + List routingPathList = IndexMetadata.INDEX_DIMENSIONS.get(result); assertEquals(3, routingPathList.size()); } @@ -552,10 +564,10 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); assertThat( - IndexMetadata.INDEX_ROUTING_PATH.get(result), + IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*") ); - List routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result); + List routingPathList = IndexMetadata.INDEX_DIMENSIONS.get(result); assertEquals(3, routingPathList.size()); } @@ -605,7 +617,7 @@ public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch() assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); } public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() throws Exception { @@ -652,8 +664,8 @@ public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() thro Settings result = generateTsdbSettings(mapping, now); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); - assertEquals(2, IndexMetadata.INDEX_ROUTING_PATH.get(result).size()); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + assertEquals(2, IndexMetadata.INDEX_DIMENSIONS.get(result).size()); } public void testGenerateRoutingPathFromPassThroughObject() throws Exception { @@ -665,7 +677,12 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception { "labels": { "type": "passthrough", "time_series_dimension": true, - "priority": 2 + "priority": 2, + "properties": { + "label1": { + "type": "keyword" + } + } }, "metrics": { "type": "passthrough", @@ -683,7 +700,7 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception { assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("labels.*")); } private Settings generateTsdbSettings(String mapping, Instant now) throws IOException { From a1be48b3c3aa1c915fe750e29dbc5953ed3386da Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 11 Aug 2025 08:25:25 +0200 Subject: [PATCH 06/62] Update index.dimensions on mapping updates Fall back to index.routing_path if the dimensions can't be identified by a simple path math --- .../datastreams/TSDBIndexingIT.java | 209 +++++++++++++++++- .../DataStreamIndexSettingsProvider.java | 119 ++++++---- .../DataStreamIndexSettingsProviderTests.java | 3 +- .../cluster/metadata/IndexMetadata.java | 16 +- .../metadata/MetadataMappingService.java | 41 +++- .../index/IndexSettingProvider.java | 5 + .../index/mapper/DynamicTemplate.java | 21 ++ .../snapshots/SnapshotResiliencyTests.java | 6 +- 8 files changed, 362 insertions(+), 58 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index eaef99d86a86e..a17d82cb8dd06 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -14,6 +14,8 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; @@ -24,6 +26,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; @@ -34,6 +37,7 @@ import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; @@ -58,12 +62,14 @@ import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -321,7 +327,7 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception { .indexPatterns(List.of("k8s*")) .template( new Template( - Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset").build(), + Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(mappingTemplate), null ) @@ -329,7 +335,7 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception { .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build() ); - client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + assertResponse(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request), ElasticsearchAssertions::assertAcked); } public void testInvalidTsdbTemplatesMissingSettings() throws Exception { @@ -621,6 +627,205 @@ public void testReindexing() throws Exception { ); } + public void testAddDimensionToMapping() throws Exception { + String dataStreamName = "my-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .template( + new Template( + Settings.builder().put("index.mode", "time_series").build(), + new CompressedXContent(MAPPING_TEMPLATE), + null + ) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + // create data stream + CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "my-ds" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), equalTo(List.of("metricset"))); + + // put mapping with k8s.pod.uid as another time series dimension + var putMappingRequest = new PutMappingRequest(dataStreamName).source(""" + { + "properties": { + "k8s.pod.name": { + "type": "keyword", + "time_series_dimension": true + } + } + } + """, XContentType.JSON); + assertAcked(client().execute(TransportPutMappingAction.TYPE, putMappingRequest).actionGet()); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name")); + + indexWithPodNames(dataStreamName, Instant.now(), Map.of(), "dog", "cat"); + } + + public void testDynamicStringDimensions() throws Exception { + String dataStreamName = "my-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .template( + new Template( + Settings.builder().put("index.mode", "time_series").build(), + new CompressedXContent(""" + + { + "_doc": { + "dynamic_templates": [ + { + "labels": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword", + "time_series_dimension": true + } + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""), + null + ) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "my-ds" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); + + // doesn't populate index.dimensions because the "labels" dynamic template doesn't have a path_math + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + + // index doc + BulkResponse bulkResponse = client().prepareBulk() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .add( + client().prepareIndex(dataStreamName) + .setOpType(DocWriteRequest.OpType.CREATE) + .setSource(DOC.replace("$time", formatInstant(Instant.now())), XContentType.JSON) + ) + .get(); + assertThat(bulkResponse.hasFailures(), is(false)); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + } + + public void testDynamicDimensions() throws Exception { + String dataStreamName = "my-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .template( + new Template( + Settings.builder().put("index.mode", "time_series").build(), + new CompressedXContent(""" + + { + "_doc": { + "dynamic_templates": [ + { + "label": { + "mapping": { + "type": "keyword", + "time_series_dimension": true + } + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""), + null + ) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "my-ds" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); + + // doesn't populate index.dimensions because the "label" dynamic template doesn't have a path_math + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + + // index doc + indexWithPodNames(dataStreamName, Instant.now(), Map.of("k8s.pod.name", "label"), "dog", "cat"); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + } + + private void indexWithPodNames(String dataStreamName, Instant timestamp, Map dynamicTemplates, String... podNames) { + // index doc + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (String podName : podNames) { + bulkRequestBuilder.add( + client().prepareIndex(dataStreamName) + .setOpType(DocWriteRequest.OpType.CREATE) + .setSource(DOC.replace("$time", formatInstant(timestamp)).replace("dog", podName), XContentType.JSON) + .request() + .setDynamicTemplates(dynamicTemplates) + ); + } + + BulkResponse bulkResponse = bulkRequestBuilder.get(); + assertThat(bulkResponse.hasFailures(), is(false)); + } + + private T getSetting(String dataStreamName, Setting setting) { + GetIndexResponse getIndexResponse = safeGet( + indicesAdmin().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(dataStreamName)) + ); + assertThat(getIndexResponse.getIndices().length, equalTo(1)); + Settings settings = getIndexResponse.getSettings().get(getIndexResponse.getIndices()[0]); + return setting.get(settings); + } + + static String formatInstant(Instant instant) { return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index 9c334adcd1134..8c10266907c57 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -24,19 +24,16 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.PassThroughObjectMapper; import java.io.IOException; import java.io.UncheckedIOException; import java.time.Instant; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.Locale; @@ -120,16 +117,19 @@ public Settings getAdditionalIndexSettings( builder.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), FORMATTER.format(start)); builder.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), FORMATTER.format(end)); - if (combinedTemplateMappings.isEmpty() == false) { - List dimensions = findDimensionFields( + if (indexTemplateAndCreateRequestSettings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey()) == false + && combinedTemplateMappings.isEmpty() == false) { + List dimensions = new ArrayList<>(); + boolean matchesAllDimensions = findDimensionFields( indexName, indexTemplateAndCreateRequestSettings, - combinedTemplateMappings + combinedTemplateMappings, + dimensions ); if (dimensions.isEmpty() == false) { // TODO handle the case when adding a dimension field to the mappings of an existing index // at the moment, the index.dimensions setting is only set when an index is created - builder.putList(INDEX_DIMENSIONS.getKey(), dimensions); + builder.putList(matchesAllDimensions ? INDEX_DIMENSIONS.getKey() : INDEX_ROUTING_PATH.getKey(), dimensions); } } return builder.build(); @@ -140,6 +140,32 @@ public Settings getAdditionalIndexSettings( return Settings.EMPTY; } + /** + * This is called when mappings are updated, so that the {@link IndexMetadata#INDEX_DIMENSIONS} + * and {@link IndexMetadata#INDEX_ROUTING_PATH} settings are updated to match the new mappings. + * Updates {@link IndexMetadata#INDEX_DIMENSIONS} if a new dimension field is added to the mappings, + * or sets {@link IndexMetadata#INDEX_ROUTING_PATH} if a new dimension field is added that doesn't allow for matching all + * dimension fields via a wildcard pattern. + */ + @Override + public Settings onUpdateMappings(IndexMetadata indexMetadata, DocumentMapper documentMapper) { + List indexDimensions = INDEX_DIMENSIONS.get(indexMetadata.getSettings()); + if (indexDimensions.isEmpty()) { + return Settings.EMPTY; + } + assert indexMetadata.getIndexMode() == IndexMode.TIME_SERIES; + List newIndexDimensions = new ArrayList<>(indexDimensions.size()); + boolean matchesAllDimensions = findDimensionFields(newIndexDimensions, documentMapper); + if (indexDimensions.equals(newIndexDimensions)) { + return Settings.EMPTY; + } + if (matchesAllDimensions) { + return Settings.builder().putList(INDEX_DIMENSIONS.getKey(), newIndexDimensions).build(); + } else { + return Settings.builder().putList(INDEX_ROUTING_PATH.getKey(), newIndexDimensions).build(); + } + } + /** * Find fields in mapping that are time_series_dimension enabled. * Using MapperService here has an overhead, but allows the mappings from template to @@ -149,7 +175,12 @@ public Settings getAdditionalIndexSettings( * iterate over all values to find the field that can serve as routing value. But this requires * mapping specific logic to exist here. */ - private List findDimensionFields(String indexName, Settings allSettings, List combinedTemplateMappings) { + private boolean findDimensionFields( + String indexName, + Settings allSettings, + List combinedTemplateMappings, + List dimensions + ) { var tmpIndexMetadata = IndexMetadata.builder(indexName); int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(allSettings); @@ -173,61 +204,51 @@ private List findDimensionFields(String indexName, Settings allSettings, // Create MapperService just to extract keyword dimension fields: try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata.build())) { mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); - List routingPaths = new ArrayList<>(); - for (var objectMapper : mapperService.documentMapper().mappers().objectMappers().values()) { - if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { - if (passThroughObjectMapper.containsDimensions()) { - routingPaths.add(passThroughObjectMapper.fullPath() + ".*"); - } + DocumentMapper documentMapper = mapperService.documentMapper(); + return findDimensionFields(dimensions, documentMapper); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static boolean findDimensionFields(List dimensions, DocumentMapper documentMapper) { + for (var objectMapper : documentMapper.mappers().objectMappers().values()) { + if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { + if (passThroughObjectMapper.containsDimensions()) { + dimensions.add(passThroughObjectMapper.fullPath() + ".*"); } } - for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) { - extractPath(routingPaths, fieldMapper); + } + boolean matchesAllDimensions = true; + for (var template : documentMapper.mapping().getRoot().dynamicTemplates()) { + if (template.isTimeSeriesDimension() == false) { + continue; } - for (var template : mapperService.getAllDynamicTemplates()) { - if (template.pathMatch().isEmpty()) { - continue; - } - - var templateName = "__dynamic__" + template.name(); - var mappingSnippet = template.mappingForName(templateName, KeywordFieldMapper.CONTENT_TYPE); - String mappingSnippetType = (String) mappingSnippet.get("type"); - if (mappingSnippetType == null) { - continue; - } - - MappingParserContext parserContext = mapperService.parserContext(); - for (Iterator iterator = template.pathMatch().iterator(); iterator.hasNext();) { - var mapper = parserContext.typeParser(mappingSnippetType) - .parse(iterator.next(), mappingSnippet, parserContext) - .build(MapperBuilderContext.root(false, false)); - extractPath(routingPaths, mapper); - if (iterator.hasNext()) { - // Since FieldMapper.parse modifies the Map passed in (removing entries for "type"), that means - // that only the first pathMatch passed in gets recognized as a time_series_dimension. - // To avoid this, each parsing call uses a new mapping snippet. - // Note that a shallow copy of the mappingSnippet map is not enough if there are multi-fields. - mappingSnippet = template.mappingForName(templateName, KeywordFieldMapper.CONTENT_TYPE); - } - } + if (template.isSimplePathMath() == false) { + matchesAllDimensions = false; } - return routingPaths; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (template.pathMatch().isEmpty() == false) { + dimensions.addAll(template.pathMatch()); + } + } + + for (var fieldMapper : documentMapper.mappers().fieldMappers()) { + extractPath(dimensions, fieldMapper); } + return matchesAllDimensions; } /** * Helper method that adds the name of the mapper to the provided list. */ - private static void extractPath(List routingPaths, Mapper mapper) { + private static void extractPath(List dimensions, Mapper mapper) { if (mapper instanceof FieldMapper fieldMapper) { if (fieldMapper.fieldType().isDimension()) { String path = mapper.fullPath(); // don't add if the path already matches via a wildcard pattern in the list // e.g. if "path.*" is already added, "path.foo" should not be added - if (Regex.simpleMatch(routingPaths, path) == false) { - routingPaths.add(path); + if (Regex.simpleMatch(dimensions, path) == false) { + dimensions.add(path); } } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index 318623a81fce2..bdd94de1f92c6 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -143,11 +143,10 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: // (in production the index.mode setting is usually provided in an index or component template) result = builder().put(result).put("index.mode", "time_series").build(); - assertThat(result.size(), equalTo(4)); + assertThat(result.size(), equalTo(3)); assertThat(result.get(IndexSettings.MODE.getKey()), equalTo("time_series")); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field2", "field3")); } public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 1038ba71fa201..a4caa74d91316 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -514,10 +514,24 @@ public Iterator> settings() { Property.ServerlessPublic ); + /** + * Populated when an index that belongs to a time_series data stream is created or its mappings are updated. + * This setting is used so that the coordinating node knows which fields are time series dimensions + * as it doesn't have access to mappings. + * When this setting is populated, an optimization kicks in that allows the coordinating node to create the tsid and the routing hash + * in one go. + * Otherwise, the coordinating node only creates the routing hash based on {@link #INDEX_ROUTING_PATH} and the tsid is created + * during document parsing, effectively requiring two passes over the document. + *

+ * The condition for this optimization to kick in is that all possible dimension fields can be identified + * via a list of wildcard patterns. + * If that's not the case (for example when certain types of dynamic templates are used), + * the {@link #INDEX_ROUTING_PATH} is populated instead. + */ public static final Setting> INDEX_DIMENSIONS = Setting.stringListSetting( "index.dimensions", Setting.Property.IndexScope, - Setting.Property.Final + Property.Dynamic ); /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java index 6680b804dc25b..4094c58f499ee 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java @@ -24,9 +24,12 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; @@ -39,6 +42,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * Service responsible for submitting mapping changes @@ -53,10 +57,14 @@ public class MetadataMappingService { private final MasterServiceTaskQueue taskQueue; @Inject - public MetadataMappingService(ClusterService clusterService, IndicesService indicesService) { + public MetadataMappingService( + ClusterService clusterService, + IndicesService indicesService, + IndexSettingProviders indexSettingProviders + ) { this.clusterService = clusterService; this.indicesService = indicesService; - this.taskQueue = clusterService.createTaskQueue("put-mapping", Priority.HIGH, new PutMappingExecutor()); + this.taskQueue = clusterService.createTaskQueue("put-mapping", Priority.HIGH, new PutMappingExecutor(indexSettingProviders)); } record PutMappingClusterStateUpdateTask(PutMappingClusterStateUpdateRequest request, ActionListener listener) @@ -96,6 +104,16 @@ public TimeValue ackTimeout() { } class PutMappingExecutor implements ClusterStateTaskExecutor { + private final IndexSettingProviders indexSettingProviders; + + PutMappingExecutor() { + this(new IndexSettingProviders(Set.of())); + } + + PutMappingExecutor(IndexSettingProviders indexSettingProviders) { + this.indexSettingProviders = indexSettingProviders; + } + @Override public ClusterState execute(BatchExecutionContext batchExecutionContext) throws Exception { Map indexMapperServices = new HashMap<>(); @@ -126,7 +144,7 @@ public ClusterState execute(BatchExecutionContext indexMapperServices @@ -200,9 +218,26 @@ private static ClusterState applyRequest( indexMetadataBuilder.putMapping(new MappingMetadata(docMapper)); indexMetadataBuilder.putInferenceFields(docMapper.mappers().inferenceFields()); } + boolean updatedSettings = false; + final Settings.Builder additionalIndexSettings = Settings.builder(); if (updatedMapping) { indexMetadataBuilder.mappingVersion(1 + indexMetadataBuilder.mappingVersion()) .mappingsUpdatedVersion(IndexVersion.current()); + for (IndexSettingProvider provider : indexSettingProviders.getIndexSettingProviders()) { + Settings newAdditionalSettings = provider.onUpdateMappings(indexMetadata, docMapper); + if (newAdditionalSettings != null && newAdditionalSettings.isEmpty() == false) { + MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, additionalIndexSettings); + additionalIndexSettings.put(newAdditionalSettings); + updatedSettings = true; + } + } + } + if (updatedSettings) { + final Settings.Builder indexSettingsBuilder = Settings.builder(); + indexSettingsBuilder.put(indexMetadata.getSettings()); + indexSettingsBuilder.put(additionalIndexSettings.build()); + indexMetadataBuilder.settings(indexSettingsBuilder.build()); + indexMetadataBuilder.settingsVersion(1 + indexMetadata.getSettingsVersion()); } /* * This implicitly increments the index metadata version and builds the index metadata. This means that we need to have diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java index ee2657a16393a..1bf9a351e11f8 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; @@ -52,6 +53,10 @@ Settings getAdditionalIndexSettings( List combinedTemplateMappings ); + default Settings onUpdateMappings(IndexMetadata indexMetadata, DocumentMapper indexMetadataBuilder) { + return Settings.EMPTY; + } + /** * Infrastructure class that holds services that can be used by {@link IndexSettingProvider} instances. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index 7c58e58390503..3f8a79d27986c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.function.Predicate; @@ -31,6 +32,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.elasticsearch.index.mapper.TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM; + public class DynamicTemplate implements ToXContentObject { public enum MatchType { @@ -447,6 +450,24 @@ public List match() { return match; } + public boolean isTimeSeriesDimension() { + return Optional.of(mapping) + .flatMap(m -> Optional.ofNullable(m.get(TIME_SERIES_DIMENSION_PARAM))) + .filter(Boolean.class::isInstance) + .map(Boolean.class::cast) + .orElse(false); + } + + public boolean isSimplePathMath() { + return pathMatch.isEmpty() == false + && pathUnmatch.isEmpty() + && match.isEmpty() + && unmatch.isEmpty() + && matchMappingType.isEmpty() + && unmatchMappingType.isEmpty() + && matchType != MatchType.REGEX; + } + public boolean match(String templateName, String path, String fieldName, XContentFieldType xcontentFieldType) { // If the template name parameter is specified, then we will check only the name of the template and ignore other matches. if (templateName != null) { diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 832f4d5f890bb..f2eb321c3cb9e 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -2541,7 +2541,11 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { VerifyNodeRepositoryCoordinationAction.TYPE, new VerifyNodeRepositoryCoordinationAction.LocalAction(actionFilters, transportService, clusterService, client) ); - final MetadataMappingService metadataMappingService = new MetadataMappingService(clusterService, indicesService); + final MetadataMappingService metadataMappingService = new MetadataMappingService( + clusterService, + indicesService, + new IndexSettingProviders(Set.of()) + ); peerRecoverySourceService = new PeerRecoverySourceService( transportService, From 1734649dc8ee71ad705f0c8bbe0df7b70a1795a6 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 11 Aug 2025 08:26:23 +0200 Subject: [PATCH 07/62] Apply spotless suggestions --- .../datastreams/TSDBIndexingIT.java | 117 ++++++++---------- 1 file changed, 50 insertions(+), 67 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index a17d82cb8dd06..50e612903248c 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -326,11 +326,7 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception { ComposableIndexTemplate.builder() .indexPatterns(List.of("k8s*")) .template( - new Template( - Settings.builder().put("index.mode", "time_series").build(), - new CompressedXContent(mappingTemplate), - null - ) + new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(mappingTemplate), null) ) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build() @@ -679,38 +675,32 @@ public void testDynamicStringDimensions() throws Exception { putTemplateRequest.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(List.of(dataStreamName)) - .template( - new Template( - Settings.builder().put("index.mode", "time_series").build(), - new CompressedXContent(""" - - { - "_doc": { - "dynamic_templates": [ - { - "labels": { - "match_mapping_type": "string", - "mapping": { - "type": "keyword", - "time_series_dimension": true - } - } - } - ], - "properties": { - "@timestamp": { - "type": "date" - }, - "metricset": { - "type": "keyword", - "time_series_dimension": true - } - } + .template(new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(""" + + { + "_doc": { + "dynamic_templates": [ + { + "labels": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword", + "time_series_dimension": true } - }"""), - null - ) - ) + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""), null)) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build() ); @@ -746,37 +736,31 @@ public void testDynamicDimensions() throws Exception { putTemplateRequest.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(List.of(dataStreamName)) - .template( - new Template( - Settings.builder().put("index.mode", "time_series").build(), - new CompressedXContent(""" - - { - "_doc": { - "dynamic_templates": [ - { - "label": { - "mapping": { - "type": "keyword", - "time_series_dimension": true - } - } - } - ], - "properties": { - "@timestamp": { - "type": "date" - }, - "metricset": { - "type": "keyword", - "time_series_dimension": true - } - } + .template(new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(""" + + { + "_doc": { + "dynamic_templates": [ + { + "label": { + "mapping": { + "type": "keyword", + "time_series_dimension": true } - }"""), - null - ) - ) + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""), null)) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build() ); @@ -825,7 +809,6 @@ private T getSetting(String dataStreamName, Setting setting) { return setting.get(settings); } - static String formatInstant(Instant instant) { return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); } From d449b79c0d30476c9e20d2e31e6b86ca378f0f57 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 11 Aug 2025 09:34:26 +0200 Subject: [PATCH 08/62] Make index.dimensions a private setting --- .../cluster/metadata/IndexMetadata.java | 3 +- .../metadata/MetadataCreateIndexService.java | 19 ++++++++--- .../MetadataIndexTemplateService.java | 34 +++++++++++++------ 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index a4caa74d91316..dfac3a21cd4c3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -531,7 +531,8 @@ public Iterator> settings() { public static final Setting> INDEX_DIMENSIONS = Setting.stringListSetting( "index.dimensions", Setting.Property.IndexScope, - Property.Dynamic + Property.Dynamic, + Property.PrivateIndex ); /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index ca612eb20747a..d945a9967cc53 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1566,7 +1566,7 @@ private void validate(CreateIndexClusterStateUpdateRequest request, ProjectMetad public void validateIndexSettings(String indexName, final Settings settings, final boolean forbidPrivateIndexSettings) throws IndexCreationException { - List validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings); + List validationErrors = getIndexSettingsValidationErrors(settings, null, forbidPrivateIndexSettings); if (validationErrors.isEmpty() == false) { ValidationException validationException = new ValidationException(); @@ -1575,21 +1575,30 @@ public void validateIndexSettings(String indexName, final Settings settings, fin } } - List getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings) { + List getIndexSettingsValidationErrors( + final Settings settings, + @Nullable Settings systemProvided, + final boolean forbidPrivateIndexSettings + ) { List validationErrors = validateIndexCustomPath(settings, env.sharedDataDir()); if (forbidPrivateIndexSettings) { - validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings)); + validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, systemProvided, indexScopedSettings)); } return validationErrors; } - private static List validatePrivateSettingsNotExplicitlySet(Settings settings, IndexScopedSettings indexScopedSettings) { + private static List validatePrivateSettingsNotExplicitlySet( + Settings settings, + @Nullable Settings systemProvided, + IndexScopedSettings indexScopedSettings + ) { List validationErrors = new ArrayList<>(); for (final String key : settings.keySet()) { final Setting setting = indexScopedSettings.get(key); if (setting == null) { assert indexScopedSettings.isPrivateSetting(key) : "expected [" + key + "] to be private but it was not"; - } else if (setting.isPrivateIndex()) { + } else if (setting.isPrivateIndex() && systemProvided != null && settings.get(key).equals(systemProvided.get(key)) == false) { + // if the setting is system provided, they're not set by the user validationErrors.add("private index setting [" + key + "] can not be set explicitly"); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index ccddfd52b95bf..875e1b1cc4f88 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -786,6 +786,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo final var combinedSettings = resolveSettings(indexTemplate, projectMetadata.componentTemplates()); // First apply settings sourced from index setting providers: var finalSettings = Settings.builder(); + var additionalSettingsBuilder = Settings.builder(); for (var provider : indexSettingProviders) { var newAdditionalSettings = provider.getAdditionalIndexSettings( VALIDATE_INDEX_NAME, @@ -796,9 +797,11 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo combinedSettings, combinedMappings ); - MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, finalSettings); - finalSettings.put(newAdditionalSettings); + MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, additionalSettingsBuilder); + additionalSettingsBuilder.put(newAdditionalSettings); } + Settings additionalSettings = additionalSettingsBuilder.build(); + finalSettings.put(additionalSettings); // Then apply setting from component templates: finalSettings.put(combinedSettings); // Then finally apply settings resolved from index template: @@ -808,7 +811,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo var templateToValidate = indexTemplate.toBuilder().template(Template.builder(finalTemplate).settings(finalSettings)).build(); - validate(name, templateToValidate); + validate(name, templateToValidate, additionalSettings); validateDataStreamsStillReferenced(projectMetadata, name, templateToValidate); validateLifecycle(projectMetadata, name, templateToValidate, globalRetentionSettings.get(false)); validateDataStreamOptions(projectMetadata, name, templateToValidate, globalRetentionSettings.get(true)); @@ -2037,18 +2040,19 @@ static void validateTemplate(Settings validateSettings, CompressedXContent mappi } private void validate(String name, ComponentTemplate template) { - validate(name, template.template(), Collections.emptyList()); + validate(name, template.template(), Collections.emptyList(), null); } - private void validate(String name, ComposableIndexTemplate template) { - validate(name, template.template(), template.indexPatterns()); + private void validate(String name, ComposableIndexTemplate template, Settings additionalSettings) { + validate(name, template.template(), template.indexPatterns(), additionalSettings); } - private void validate(String name, Template template, List indexPatterns) { + private void validate(String name, Template template, List indexPatterns, @Nullable Settings systemProvided) { Optional