diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 874b2316406ef..47defea0a1f95 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -56,11 +56,13 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.OnScriptError; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; @@ -808,13 +810,18 @@ private static Response prepareRamIndex( try (IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(defaultAnalyzer))) { BytesReference document = request.contextSetup.document; XContentType xContentType = request.contextSetup.xContentType; - String id; - if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { - id = null; // The id gets auto generated for time series indices. - } else { - id = "_id"; - } - SourceToParse sourceToParse = new SourceToParse(id, document, xContentType); + + SourceToParse sourceToParse = (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) + ? new SourceToParse( + null, + document, + xContentType, + indexService.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID) + ? TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE + : null + ) + : new SourceToParse("_id", document, xContentType); + DocumentMapper documentMapper = indexService.mapperService().documentMapper(); if (documentMapper == null) { documentMapper = DocumentMapper.createEmpty(indexService.mapperService()); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 508e438932e68..e1df6c130c9fe 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -293,7 +293,7 @@ public void parse(DocumentParserContext context) throws IOException { if (parent == null) { throw new IllegalArgumentException("[parent] is missing for join field [" + name() + "]"); } - if (context.sourceToParse().routing() == null) { + if (context.routing() == null) { throw new IllegalArgumentException("[routing] is missing for join field [" + name() + "]"); } String fieldName = fieldType().joiner.parentJoinField(name); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 70168f6a2b516..fe7af4bc26e6e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -689,9 +689,7 @@ private static Engine.Result performOpOnReplica( indexRequest.id(), indexRequest.source(), indexRequest.getContentType(), - indexRequest.routing(), - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE + indexRequest.routing() ); result = replica.applyIndexOperationOnReplica( primaryResponse.getSeqNo(), diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 5bdd197b80d2c..a8d6220415a43 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -881,7 +881,7 @@ public Index getConcreteWriteIndex(IndexAbstraction ia, Metadata metadata) { @Override public int route(IndexRouting indexRouting) { - return indexRouting.indexShard(id, routing, contentType, source); + return indexRouting.indexShard(id, routing, contentType, source, this::routing); } public IndexRequest setRequireAlias(boolean requireAlias) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 1ed9d759c4ca8..fb2fcf1a02ad0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -21,6 +21,8 @@ import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParser.Token; @@ -35,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Consumer; import java.util.function.IntConsumer; import java.util.function.IntSupplier; import java.util.function.Predicate; @@ -74,7 +77,13 @@ private IndexRouting(IndexMetadata metadata) { * Called when indexing a document to generate the shard id that should contain * a document with the provided parameters. */ - public abstract int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source); + public abstract int indexShard( + String id, + @Nullable String routing, + XContentType sourceType, + BytesReference source, + Consumer routingHashSetter + ); /** * Called when updating a document to generate the shard id that should contain @@ -153,7 +162,13 @@ public void process(IndexRequest indexRequest) { } @Override - public int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source) { + public int indexShard( + String id, + @Nullable String routing, + XContentType sourceType, + BytesReference source, + Consumer routingHashSetter + ) { if (id == null) { throw new IllegalStateException("id is required and should have been set by process"); } @@ -237,12 +252,14 @@ public void collectSearchShards(String routing, IntConsumer consumer) { public static class ExtractFromSource extends IndexRouting { private final Predicate isRoutingPath; private final XContentParserConfiguration parserConfig; + private final boolean trackTimeSeriesRoutingHash; ExtractFromSource(IndexMetadata metadata) { super(metadata); if (metadata.isRoutingPartitionedIndex()) { throw new IllegalArgumentException("routing_partition_size is incompatible with routing_path"); } + trackTimeSeriesRoutingHash = metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID); List routingPaths = metadata.getRoutingPaths(); isRoutingPath = Regex.simpleMatcher(routingPaths.toArray(String[]::new)); this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true); @@ -256,10 +273,20 @@ public boolean matchesField(String fieldName) { public void process(IndexRequest indexRequest) {} @Override - public int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source) { + public int indexShard( + String id, + @Nullable String routing, + XContentType sourceType, + BytesReference source, + Consumer routingHashSetter + ) { assert Transports.assertNotTransportThread("parsing the _source can get slow"); checkNoRouting(routing); - return hashToShardId(hashSource(sourceType, source).buildHash(IndexRouting.ExtractFromSource::defaultOnEmpty)); + int hash = hashSource(sourceType, source).buildHash(IndexRouting.ExtractFromSource::defaultOnEmpty); + if (trackTimeSeriesRoutingHash) { + routingHashSetter.accept(TimeSeriesRoutingHashFieldMapper.encode(hash)); + } + return hashToShardId(hash); } public String createId(XContentType sourceType, BytesReference source, byte[] suffix) { @@ -334,16 +361,13 @@ private void extractItem(String path, XContentParser source) throws IOException source.nextToken(); break; case VALUE_STRING: + case VALUE_NUMBER: hashes.add(new NameAndHash(new BytesRef(path), hash(new BytesRef(source.text())))); source.nextToken(); break; case VALUE_NULL: source.nextToken(); break; - case VALUE_NUMBER: // allow parsing numbers assuming routing fields are always keyword fields - hashes.add(new NameAndHash(new BytesRef(path), hash(new BytesRef(source.text())))); - source.nextToken(); - break; default: throw new ParsingException( source.getTokenLocation(), diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 05afc14e0f0cd..05169836d6617 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; import java.io.IOException; @@ -92,6 +93,12 @@ public MetadataFieldMapper timeSeriesIdFieldMapper() { return null; } + @Override + public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { + // non time-series indices must not have a TimeSeriesRoutingIdFieldMapper + return null; + } + @Override public IdFieldMapper idFieldMapperWithoutFieldData() { return ProvidedIdFieldMapper.NO_FIELD_DATA; @@ -185,6 +192,11 @@ public MetadataFieldMapper timeSeriesIdFieldMapper() { return TimeSeriesIdFieldMapper.INSTANCE; } + @Override + public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { + return TimeSeriesRoutingHashFieldMapper.INSTANCE; + } + public IdFieldMapper idFieldMapperWithoutFieldData() { return TsidExtractingIdFieldMapper.INSTANCE; } @@ -322,6 +334,13 @@ public String getName() { */ public abstract MetadataFieldMapper timeSeriesIdFieldMapper(); + /** + * Return an instance of the {@link TimeSeriesRoutingHashFieldMapper} that generates + * the _ts_routing_hash field. The field mapper will be added to the list of the metadata + * field mappers for the index. + */ + public abstract MetadataFieldMapper timeSeriesRoutingHashFieldMapper(); + /** * How {@code time_series_dimension} fields are handled by indices in this mode. */ diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 0ddcef2ac3a08..bca7b963becaa 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -102,6 +102,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion UPGRADE_LUCENE_9_9_2 = def(8_502_00_0, Version.LUCENE_9_9_2); public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_502_00_1, Version.LUCENE_9_9_2); public static final IndexVersion UPGRADE_TO_LUCENE_9_10 = def(8_503_00_0, Version.LUCENE_9_10_0); + public static final IndexVersion TIME_SERIES_ROUTING_HASH_IN_ID = def(8_504_00_0, Version.LUCENE_9_10_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java index 2a2ae7245d996..ae497af887d9c 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java @@ -115,8 +115,8 @@ boolean useTSDBDocValuesFormat(final String field) { private boolean excludeFields(String fieldName) { // Avoid using tsdb codec for fields like _seq_no, _primary_term. - // But _tsid should always use the tesbd codec. - return fieldName.startsWith("_") && fieldName.equals("_tsid") == false; + // But _tsid and _ts_routing_hash should always use the tsdb codec. + return fieldName.startsWith("_") && fieldName.equals("_tsid") == false && fieldName.equals("_ts_routing_hash") == false; } private boolean isTimeSeriesModeIndex() { diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java index e5eeac72927c0..e054fc52b562e 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java @@ -59,11 +59,9 @@ import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import java.io.IOException; import java.util.Collections; -import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -254,14 +252,7 @@ private LeafReader getDelegate() { private LeafReader createInMemoryLeafReader() { assert Thread.holdsLock(this); final ParsedDocument parsedDocs = documentParser.parseDocument( - new SourceToParse( - operation.id(), - operation.source(), - XContentHelper.xContentType(operation.source()), - operation.routing(), - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE - ), + new SourceToParse(operation.id(), operation.source(), XContentHelper.xContentType(operation.source()), operation.routing()), mappingLookup ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 9a0e391102708..1fda9ababfabd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -99,7 +99,7 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL context.version(), context.seqID(), context.id(), - source.routing(), + context.routing(), context.reorderParentAndGetDocs(), context.sourceToParse().source(), context.sourceToParse().getXContentType(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 01e67377adafd..92aa8662eaf9d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -12,6 +12,7 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexableField; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.xcontent.FilterXContentParserWrapper; @@ -231,6 +232,10 @@ public final SourceToParse sourceToParse() { return this.sourceToParse; } + public final String routing() { + return mappingParserContext.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? null : sourceToParse.routing(); + } + /** * Add the given {@code field} to the set of ignored fields. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index c965a77f1b5bf..ef15af93f6e34 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -66,21 +66,24 @@ final class TsIdLoader implements IdLoader { } public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException { - IndexRouting.ExtractFromSource.Builder[] builders = new IndexRouting.ExtractFromSource.Builder[docIdsInLeaf.length]; - for (int i = 0; i < builders.length; i++) { - builders[i] = indexRouting.builder(); - } + IndexRouting.ExtractFromSource.Builder[] builders = null; + if (indexRouting != null) { + builders = new IndexRouting.ExtractFromSource.Builder[docIdsInLeaf.length]; + for (int i = 0; i < builders.length; i++) { + builders[i] = indexRouting.builder(); + } - for (String routingField : routingPaths) { - // Routing field must always be keyword fields, so it is ok to use SortedSetDocValues directly here. - SortedSetDocValues dv = DocValues.getSortedSet(reader, routingField); - for (int i = 0; i < docIdsInLeaf.length; i++) { - int docId = docIdsInLeaf[i]; - var builder = builders[i]; - if (dv.advanceExact(docId)) { - for (int j = 0; j < dv.docValueCount(); j++) { - BytesRef routingValue = dv.lookupOrd(dv.nextOrd()); - builder.addMatching(routingField, routingValue); + for (String routingField : routingPaths) { + // Routing field must always be keyword fields, so it is ok to use SortedSetDocValues directly here. + SortedSetDocValues dv = DocValues.getSortedSet(reader, routingField); + for (int i = 0; i < docIdsInLeaf.length; i++) { + int docId = docIdsInLeaf[i]; + var builder = builders[i]; + if (dv.advanceExact(docId)) { + for (int j = 0; j < dv.docValueCount(); j++) { + BytesRef routingValue = dv.lookupOrd(dv.nextOrd()); + builder.addMatching(routingField, routingValue); + } } } } @@ -100,9 +103,17 @@ public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] assert found; assert timestampDocValues.docValueCount() == 1; long timestamp = timestampDocValues.nextValue(); - - var routingBuilder = builders[i]; - ids[i] = TsidExtractingIdFieldMapper.createId(false, routingBuilder, tsid, timestamp, new byte[16]); + if (builders != null) { + var routingBuilder = builders[i]; + ids[i] = TsidExtractingIdFieldMapper.createId(false, routingBuilder, tsid, timestamp, new byte[16]); + } else { + SortedDocValues routingHashDocValues = DocValues.getSorted(reader, TimeSeriesRoutingHashFieldMapper.NAME); + found = routingHashDocValues.advanceExact(docId); + assert found; + BytesRef routingHashBytes = routingHashDocValues.lookupOrd(routingHashDocValues.ordValue()); + int routingHash = TimeSeriesRoutingHashFieldMapper.decode(Uid.decodeId(routingHashBytes.bytes)); + ids[i] = TsidExtractingIdFieldMapper.createId(routingHash, tsid, timestamp); + } } return new TsIdLeaf(docIdsInLeaf, ids); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 3141b73174897..39686c3f30555 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -108,7 +108,7 @@ public boolean required() { @Override public void preParse(DocumentParserContext context) { - String routing = context.sourceToParse().routing(); + String routing = context.routing(); if (routing != null) { context.doc().add(new StringField(fieldType().name(), routing, Field.Store.YES)); context.addToFieldNames(fieldType().name()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java index 12f74263a3bd7..6a020127019f5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java @@ -52,6 +52,10 @@ public SourceToParse(String id, BytesReference source, XContentType xContentType this(id, source, xContentType, null, Map.of(), DocumentSizeObserver.EMPTY_INSTANCE); } + public SourceToParse(String id, BytesReference source, XContentType xContentType, String routing) { + this(id, source, xContentType, routing, Map.of(), DocumentSizeObserver.EMPTY_INSTANCE); + } + public BytesReference source() { return this.source; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 1ee7caff497ad..2d330e433d444 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -140,7 +140,13 @@ public void postParse(DocumentParserContext context) throws IOException { ? timeSeriesIdBuilder.buildLegacyTsid().toBytesRef() : timeSeriesIdBuilder.buildTsidHash().toBytesRef(); context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); - TsidExtractingIdFieldMapper.createField(context, timeSeriesIdBuilder.routingBuilder, timeSeriesId); + TsidExtractingIdFieldMapper.createField( + context, + getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID) + ? timeSeriesIdBuilder.routingBuilder + : null, + timeSeriesId + ); } private IndexVersion getIndexVersionCreated(final DocumentParserContext context) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java new file mode 100644 index 0000000000000..090fe7839b3e9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.FieldDataContext; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.plain.SortedOrdinalsIndexFieldData; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.script.field.DelegateDocValuesField; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; + +import java.util.Base64; +import java.util.Collections; + +/** + * Mapper for the {@code _ts_routing_hash} field. + * + * The field contains the routing hash, as calculated in coordinating nodes for docs in time-series indexes. + * It's stored to be retrieved and added as a prefix when reconstructing the _id field in search queries. + * The prefix can then used for routing Get and Delete requests (by doc id) to the right shard. + */ +public class TimeSeriesRoutingHashFieldMapper extends MetadataFieldMapper { + + public static final String NAME = "_ts_routing_hash"; + + public static final TimeSeriesRoutingHashFieldMapper INSTANCE = new TimeSeriesRoutingHashFieldMapper(); + + public static final TypeParser PARSER = new FixedTypeParser(c -> c.getIndexSettings().getMode().timeSeriesRoutingHashFieldMapper()); + + static final class TimeSeriesRoutingHashFieldType extends MappedFieldType { + + private static final TimeSeriesRoutingHashFieldType INSTANCE = new TimeSeriesRoutingHashFieldType(); + + private TimeSeriesRoutingHashFieldType() { + super(NAME, false, false, true, TextSearchInfo.NONE, Collections.emptyMap()); + } + + @Override + public String typeName() { + return NAME; + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, MappedFieldType.FielddataOperation.SEARCH)); + } + + @Override + public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { + failIfNoDocValues(); + return new SortedOrdinalsIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + (dv, n) -> new DelegateDocValuesField( + new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))), + n + ) + ); + } + + @Override + public Query termQuery(Object value, SearchExecutionContext context) { + throw new IllegalArgumentException("[" + NAME + "] is not searchable"); + } + } + + private TimeSeriesRoutingHashFieldMapper() { + super(TimeSeriesRoutingHashFieldType.INSTANCE); + } + + @Override + public void postParse(DocumentParserContext context) { + if (context.indexSettings().getMode() == IndexMode.TIME_SERIES + && context.indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) { + String routingHash = context.sourceToParse().routing(); + var field = new SortedDocValuesField(NAME, Uid.encodeId(routingHash != null ? routingHash : encode(0))); + context.rootDoc().add(field); + } + } + + @Override + protected String contentType() { + return NAME; + } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } + + public static String encode(int routingId) { + byte[] bytes = new byte[4]; + ByteUtils.writeIntLE(routingId, bytes, 0); + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + + public static final String DUMMY_ENCODED_VALUE = encode(0); + + public static int decode(String routingId) { + byte[] bytes = Base64.getUrlDecoder().decode(routingId); + return ByteUtils.readIntLE(bytes, 0); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 1e613767c2c89..8101b5be1b60e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; +import java.util.Base64; import java.util.Locale; /** @@ -52,22 +53,37 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra ); } long timestamp = timestampField.numericValue().longValue(); - byte[] suffix = new byte[16]; - String id = createId(context.hasDynamicMappers(), routingBuilder, tsid, timestamp, suffix); - /* - * Make sure that _id from extracting the tsid matches that _id - * from extracting the _source. This should be true for all valid - * documents with valid mappings. *But* some invalid mappings - * will not parse the field but be rejected later by the dynamic - * mappings machinery. So if there are any dynamic mappings - * at all we just skip the assertion because we can't be sure - * it always must pass. - */ - IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); - assert context.getDynamicMappers().isEmpty() == false - || context.getDynamicRuntimeFields().isEmpty() == false - || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); - + String id; + if (routingBuilder != null) { + byte[] suffix = new byte[16]; + id = createId(context.hasDynamicMappers(), routingBuilder, tsid, timestamp, suffix); + /* + * Make sure that _id from extracting the tsid matches that _id + * from extracting the _source. This should be true for all valid + * documents with valid mappings. *But* some invalid mappings + * will not parse the field but be rejected later by the dynamic + * mappings machinery. So if there are any dynamic mappings + * at all we just skip the assertion because we can't be sure + * it always must pass. + */ + IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); + assert context.getDynamicMappers().isEmpty() == false + || context.getDynamicRuntimeFields().isEmpty() == false + || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); + } else if (context.sourceToParse().routing() != null) { + int routingHash = TimeSeriesRoutingHashFieldMapper.decode(context.sourceToParse().routing()); + id = createId(routingHash, tsid, timestamp); + } else { + if (context.sourceToParse().id() == null) { + throw new IllegalArgumentException( + "_ts_routing_hash was null but must be set because index [" + + context.indexSettings().getIndexMetadata().getIndex().getName() + + "] is in time_series mode" + ); + } + // In Translog operations, the id has already been generated based on the routing hash while the latter is no longer available. + id = context.sourceToParse().id(); + } if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { throw new IllegalArgumentException( String.format( @@ -85,6 +101,18 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra context.doc().add(new StringField(NAME, uidEncoded, Field.Store.YES)); } + public static String createId(int routingHash, BytesRef tsid, long timestamp) { + Hash128 hash = new Hash128(); + MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash); + + byte[] bytes = new byte[20]; + ByteUtils.writeIntLE(routingHash, bytes, 0); + ByteUtils.writeLongLE(hash.h1, bytes, 4); + ByteUtils.writeLongBE(timestamp, bytes, 12); // Big Ending shrinks the inverted index by ~37% + + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + public static String createId( boolean dynamicMappersExists, IndexRouting.ExtractFromSource.Builder routingBuilder, diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 3bafe139756fd..046483a6b074f 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -144,7 +144,6 @@ import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.indices.recovery.RecoveryTarget; import org.elasticsearch.plugins.IndexStorePlugin; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.rest.RestStatus; @@ -1951,14 +1950,7 @@ private Engine.Result applyTranslogOperation(Engine engine, Translog.Operation o index.getAutoGeneratedIdTimestamp(), true, origin, - new SourceToParse( - index.id(), - index.source(), - XContentHelper.xContentType(index.source()), - index.routing(), - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE - ) + new SourceToParse(index.id(), index.source(), XContentHelper.xContentType(index.source()), index.routing()) ); } case DELETE -> { diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index b4c0b200eb143..a30249e94177e 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -41,7 +41,6 @@ import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.xcontent.XContentType; @@ -305,14 +304,7 @@ private static Fields generateTermVectors( } private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException { - SourceToParse source = new SourceToParse( - "_id_for_tv_api", - request.doc(), - request.xContentType(), - request.routing(), - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE - ); + SourceToParse source = new SourceToParse("_id_for_tv_api", request.doc(), request.xContentType(), request.routing()); DocumentParser documentParser = indexShard.mapperService().documentParser(); MappingLookup mappingLookup = indexShard.mapperService().mappingLookup(); ParsedDocument parsedDocument = documentParser.parseDocument(source, mappingLookup); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 795ed2120b098..b94c95834f65a 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -60,6 +60,7 @@ import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; @@ -247,6 +248,7 @@ private static Map initBuiltInMetadataMa builtInMetadataMappers.put(IdFieldMapper.NAME, IdFieldMapper.PARSER); builtInMetadataMappers.put(RoutingFieldMapper.NAME, RoutingFieldMapper.PARSER); builtInMetadataMappers.put(TimeSeriesIdFieldMapper.NAME, TimeSeriesIdFieldMapper.PARSER); + builtInMetadataMappers.put(TimeSeriesRoutingHashFieldMapper.NAME, TimeSeriesRoutingHashFieldMapper.PARSER); builtInMetadataMappers.put(IndexFieldMapper.NAME, IndexFieldMapper.PARSER); builtInMetadataMappers.put(SourceFieldMapper.NAME, SourceFieldMapper.PARSER); builtInMetadataMappers.put(NestedPathFieldMapper.NAME, NestedPathFieldMapper.PARSER); diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 7f9e808db9560..0e6800b9c8d48 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -897,20 +898,24 @@ public SourceLoader newSourceLoader() { @Override public IdLoader newIdLoader() { if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { - var indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting(); - List routingPaths = indexService.getMetadata().getRoutingPaths(); - for (String routingField : routingPaths) { - if (routingField.contains("*")) { - // In case the routing fields include path matches, find any matches and add them as distinct fields - // to the routing path. - Set matchingRoutingPaths = new TreeSet<>(routingPaths); - for (Mapper mapper : indexService.mapperService().mappingLookup().fieldMappers()) { - if (mapper instanceof KeywordFieldMapper && indexRouting.matchesField(mapper.name())) { - matchingRoutingPaths.add(mapper.name()); + IndexRouting.ExtractFromSource indexRouting = null; + List routingPaths = null; + if (indexService.getIndexSettings().getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) { + indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting(); + routingPaths = indexService.getMetadata().getRoutingPaths(); + for (String routingField : routingPaths) { + if (routingField.contains("*")) { + // In case the routing fields include path matches, find any matches and add them as distinct fields + // to the routing path. + Set matchingRoutingPaths = new TreeSet<>(routingPaths); + for (Mapper mapper : indexService.mapperService().mappingLookup().fieldMappers()) { + if (mapper instanceof KeywordFieldMapper && indexRouting.matchesField(mapper.name())) { + matchingRoutingPaths.add(mapper.name()); + } } + routingPaths = new ArrayList<>(matchingRoutingPaths); + break; } - routingPaths = new ArrayList<>(matchingRoutingPaths); - break; } } return IdLoader.createTsIdLoader(indexRouting, routingPaths); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 8af74e03f8605..d76e874c7061b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -457,7 +457,7 @@ public void testRequiredRouting() { */ private int shardIdFromSimple(IndexRouting indexRouting, String id, @Nullable String routing) { return switch (between(0, 3)) { - case 0 -> indexRouting.indexShard(id, routing, null, null); + case 0 -> indexRouting.indexShard(id, routing, null, null, null); case 1 -> indexRouting.updateShard(id, routing); case 2 -> indexRouting.deleteShard(id, routing); case 3 -> indexRouting.getShard(id, routing); @@ -490,7 +490,7 @@ public void testRoutingPathEmptySource() throws IOException { IndexRouting routing = indexRoutingForPath(between(1, 5), randomAlphaOfLength(5)); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(Map.of())) + () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(Map.of()), null) ); assertThat(e.getMessage(), equalTo("Error extracting routing: source didn't contain any routing fields")); } @@ -499,7 +499,7 @@ public void testRoutingPathMismatchSource() throws IOException { IndexRouting routing = indexRoutingForPath(between(1, 5), "foo"); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(Map.of("bar", "dog"))) + () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(Map.of("bar", "dog")), null) ); assertThat(e.getMessage(), equalTo("Error extracting routing: source didn't contain any routing fields")); } @@ -520,7 +520,7 @@ public void testRoutingIndexWithRouting() throws IOException { String docRouting = randomAlphaOfLength(5); Exception e = expectThrows( IllegalArgumentException.class, - () -> indexRouting.indexShard(randomAlphaOfLength(5), docRouting, XContentType.JSON, source) + () -> indexRouting.indexShard(randomAlphaOfLength(5), docRouting, XContentType.JSON, source, null) ); assertThat( e.getMessage(), @@ -649,7 +649,7 @@ private IndexRouting indexRoutingForPath(IndexVersion createdVersion, int shards private void assertIndexShard(IndexRouting routing, Map source, int expectedShard) throws IOException { byte[] suffix = randomSuffix(); BytesReference sourceBytes = source(source); - assertThat(routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, sourceBytes), equalTo(expectedShard)); + assertThat(routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, sourceBytes, s -> {}), equalTo(expectedShard)); IndexRouting.ExtractFromSource r = (IndexRouting.ExtractFromSource) routing; String idFromSource = r.createId(XContentType.JSON, sourceBytes, suffix); assertThat(shardIdForReadFromSourceExtracting(routing, idFromSource), equalTo(expectedShard)); diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java index d6c5fe812140f..011a23ddb0512 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java @@ -19,12 +19,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; @@ -249,14 +245,6 @@ public void testTimeSeriesLoadDocIdAndVersion() throws Exception { } private static String createTSDBId(long timestamp) { - Settings.Builder b = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field"); - IndexMetadata indexMetadata = IndexMetadata.builder("idx").settings(b).numberOfShards(1).numberOfReplicas(0).build(); - IndexRouting.ExtractFromSource.Builder routingBuilder = ((IndexRouting.ExtractFromSource) IndexRouting.fromIndexMetadata( - indexMetadata - )).builder(); - routingBuilder.addMatching("field", new BytesRef("value")); - return createId(false, routingBuilder, new BytesRef("tsid"), timestamp, new byte[16]); + return createId(randomInt(), new BytesRef("tsid"), timestamp); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java index 0f2380f6c72fb..2b8be2882c409 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java @@ -121,8 +121,9 @@ private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesR private static Set builtInMetadataFields() { Set builtInMetadataFields = new HashSet<>(IndicesModule.getBuiltInMetadataFields()); - // Index is not a time-series index, and it will not contain a _tsid field + // Index is not a time-series index, and it will not contain _tsid and _ts_routing_hash fields. builtInMetadataFields.remove(TimeSeriesIdFieldMapper.NAME); + builtInMetadataFields.remove(TimeSeriesRoutingHashFieldMapper.NAME); return builtInMetadataFields; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index 5945e5c81856f..e4ce40d4c7c29 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -46,10 +46,10 @@ public class IdLoaderTests extends ESTestCase { + private final int routingHash = randomInt(); + public void testSynthesizeIdSimple() throws Exception { - var routingPaths = List.of("dim1"); - var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); + var idLoader = IdLoader.createTsIdLoader(null, null); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs = List.of( @@ -63,17 +63,17 @@ public void testSynthesizeIdSimple() throws Exception { assertThat(leafReader.numDocs(), equalTo(3)); var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 1, 2 }); // NOTE: time series data is ordered by (tsid, timestamp) - assertThat(leaf.getId(0), equalTo(expectedId(routing, docs.get(2)))); - assertThat(leaf.getId(1), equalTo(expectedId(routing, docs.get(0)))); - assertThat(leaf.getId(2), equalTo(expectedId(routing, docs.get(1)))); + assertThat(leaf.getId(0), equalTo(expectedId(docs.get(2), routingHash))); + assertThat(leaf.getId(1), equalTo(expectedId(docs.get(0), routingHash))); + assertThat(leaf.getId(2), equalTo(expectedId(docs.get(1), routingHash))); }; - prepareIndexReader(indexAndForceMerge(routing, docs), verify, false); + prepareIndexReader(indexAndForceMerge(docs, routingHash), verify, false); } public void testSynthesizeIdMultipleSegments() throws Exception { var routingPaths = List.of("dim1"); var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); + var idLoader = IdLoader.createTsIdLoader(null, null); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs1 = List.of( @@ -96,15 +96,15 @@ public void testSynthesizeIdMultipleSegments() throws Exception { ); CheckedConsumer buildIndex = writer -> { for (Doc doc : docs1) { - indexDoc(routing, writer, doc); + indexDoc(writer, doc, routingHash); } writer.flush(); for (Doc doc : docs2) { - indexDoc(routing, writer, doc); + indexDoc(writer, doc, routingHash); } writer.flush(); for (Doc doc : docs3) { - indexDoc(routing, writer, doc); + indexDoc(writer, doc, routingHash); } writer.flush(); }; @@ -115,22 +115,22 @@ public void testSynthesizeIdMultipleSegments() throws Exception { assertThat(leafReader.numDocs(), equalTo(docs1.size())); var leaf = idLoader.leaf(null, leafReader, IntStream.range(0, docs1.size()).toArray()); for (int i = 0; i < docs1.size(); i++) { - assertThat(leaf.getId(i), equalTo(expectedId(routing, docs1.get(i)))); + assertThat(leaf.getId(i), equalTo(expectedId(docs1.get(i), routingHash))); } } { LeafReader leafReader = indexReader.leaves().get(1).reader(); assertThat(leafReader.numDocs(), equalTo(docs2.size())); var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 3 }); - assertThat(leaf.getId(0), equalTo(expectedId(routing, docs2.get(0)))); - assertThat(leaf.getId(3), equalTo(expectedId(routing, docs2.get(3)))); + assertThat(leaf.getId(0), equalTo(expectedId(docs2.get(0), routingHash))); + assertThat(leaf.getId(3), equalTo(expectedId(docs2.get(3), routingHash))); } { LeafReader leafReader = indexReader.leaves().get(2).reader(); assertThat(leafReader.numDocs(), equalTo(docs3.size())); var leaf = idLoader.leaf(null, leafReader, new int[] { 1, 2 }); - assertThat(leaf.getId(1), equalTo(expectedId(routing, docs3.get(1)))); - assertThat(leaf.getId(2), equalTo(expectedId(routing, docs3.get(2)))); + assertThat(leaf.getId(1), equalTo(expectedId(docs3.get(1), routingHash))); + assertThat(leaf.getId(2), equalTo(expectedId(docs3.get(2), routingHash))); } { LeafReader leafReader = indexReader.leaves().get(2).reader(); @@ -145,13 +145,14 @@ public void testSynthesizeIdMultipleSegments() throws Exception { public void testSynthesizeIdRandom() throws Exception { var routingPaths = List.of("dim1"); var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); + var idLoader = IdLoader.createTsIdLoader(null, null); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); Set expectedIDs = new HashSet<>(); List randomDocs = new ArrayList<>(); int numberOfTimeSeries = randomIntBetween(8, 64); for (int i = 0; i < numberOfTimeSeries; i++) { + long routingId = 0; int numberOfDimensions = randomIntBetween(1, 6); List dimensions = new ArrayList<>(numberOfDimensions); for (int j = 1; j <= numberOfDimensions; j++) { @@ -163,12 +164,13 @@ public void testSynthesizeIdRandom() throws Exception { value = randomAlphaOfLength(4); } dimensions.add(new Dimension(fieldName, value)); + routingId = value.hashCode(); } int numberOfSamples = randomIntBetween(1, 16); for (int j = 0; j < numberOfSamples; j++) { Doc doc = new Doc(startTime++, dimensions); randomDocs.add(doc); - expectedIDs.add(expectedId(routing, doc)); + expectedIDs.add(expectedId(doc, routingHash)); } } CheckedConsumer verify = indexReader -> { @@ -181,14 +183,14 @@ public void testSynthesizeIdRandom() throws Exception { assertTrue("docId=" + i + " id=" + actualId, expectedIDs.remove(actualId)); } }; - prepareIndexReader(indexAndForceMerge(routing, randomDocs), verify, false); + prepareIndexReader(indexAndForceMerge(randomDocs, routingHash), verify, false); assertThat(expectedIDs, empty()); } - private static CheckedConsumer indexAndForceMerge(IndexRouting.ExtractFromSource routing, List docs) { + private static CheckedConsumer indexAndForceMerge(List docs, int routingHash) { return writer -> { for (Doc doc : docs) { - indexDoc(routing, writer, doc); + indexDoc(writer, doc, routingHash); } writer.forceMerge(1); }; @@ -207,6 +209,7 @@ private void prepareIndexReader( } Sort sort = new Sort( new SortField(TimeSeriesIdFieldMapper.NAME, SortField.Type.STRING, false), + new SortField(TimeSeriesRoutingHashFieldMapper.NAME, SortField.Type.STRING, false), new SortedNumericSortField(DataStreamTimestampFieldMapper.DEFAULT_PATH, SortField.Type.LONG, true) ); config.setIndexSort(sort); @@ -220,8 +223,8 @@ private void prepareIndexReader( } } - private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter iw, Doc doc) throws IOException { - final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routing.builder()); + private static void indexDoc(IndexWriter iw, Doc doc, int routingHash) throws IOException { + final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); final List fields = new ArrayList<>(); fields.add(new SortedNumericDocValuesField(DataStreamTimestampFieldMapper.DEFAULT_PATH, doc.timestamp)); @@ -237,12 +240,17 @@ private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter } BytesRef tsid = builder.buildTsidHash().toBytesRef(); fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, tsid)); + fields.add( + new SortedDocValuesField( + TimeSeriesRoutingHashFieldMapper.NAME, + Uid.encodeId(TimeSeriesRoutingHashFieldMapper.encode(routingHash)) + ) + ); iw.addDocument(fields); } - private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc) throws IOException { - var routingBuilder = routing.builder(); - var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routingBuilder); + private static String expectedId(Doc doc, int routingHash) throws IOException { + var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); for (Dimension dimension : doc.dimensions) { if (dimension.value instanceof Number n) { timeSeriesIdBuilder.addLong(dimension.field, n.longValue()); @@ -250,13 +258,7 @@ private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc timeSeriesIdBuilder.addString(dimension.field, dimension.value.toString()); } } - return TsidExtractingIdFieldMapper.createId( - false, - routingBuilder, - timeSeriesIdBuilder.buildTsidHash().toBytesRef(), - doc.timestamp, - new byte[16] - ); + return TsidExtractingIdFieldMapper.createId(routingHash, timeSeriesIdBuilder.buildTsidHash().toBytesRef(), doc.timestamp); } private static IndexRouting.ExtractFromSource createRouting(List routingPaths) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index 53fcd3d331745..e0c092bfd0bfd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -12,7 +12,6 @@ import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.xcontent.XContentFactory; @@ -21,7 +20,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -53,9 +51,7 @@ public void testRoutingMapper() throws Exception { "1", BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "value").endObject()), XContentType.JSON, - "routing_value", - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE + "routing_value" ) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 94a0f2296bbfb..50abb47e51125 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -666,16 +666,44 @@ public void testParseWithDynamicMapping() { .put(IndexSettings.MODE.getKey(), "time_series") .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim") .build(); - // without _id - { - MapperService mapper = createMapperService(IndexVersion.current(), indexSettings, () -> false); - SourceToParse source = new SourceToParse(null, new BytesArray(""" - { - "@timestamp": 1609459200000, - "dim": "6a841a21", - "value": 100 - }"""), XContentType.JSON); - Engine.Index index = IndexShard.prepareIndex( + MapperService mapper = createMapperService(IndexVersion.current(), indexSettings, () -> false); + SourceToParse source = new SourceToParse(null, new BytesArray(""" + { + "@timestamp": 1609459200000, + "dim": "6a841a21", + "value": 100 + }"""), XContentType.JSON, TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE); + Engine.Index index = IndexShard.prepareIndex( + mapper, + source, + UNASSIGNED_SEQ_NO, + randomNonNegativeLong(), + Versions.MATCH_ANY, + VersionType.INTERNAL, + Engine.Operation.Origin.PRIMARY, + -1, + false, + UNASSIGNED_SEQ_NO, + 0, + System.nanoTime() + ); + assertNotNull(index.parsedDoc().dynamicMappingsUpdate()); + } + + public void testParseWithDynamicMappingInvalidRoutingHash() { + Settings indexSettings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim") + .build(); + MapperService mapper = createMapperService(IndexVersion.current(), indexSettings, () -> false); + SourceToParse source = new SourceToParse(null, new BytesArray(""" + { + "@timestamp": 1609459200000, + "dim": "6a841a21", + "value": 100 + }"""), XContentType.JSON, "no such routing hash"); + var failure = expectThrows(DocumentParsingException.class, () -> { + IndexShard.prepareIndex( mapper, source, UNASSIGNED_SEQ_NO, @@ -689,40 +717,41 @@ public void testParseWithDynamicMapping() { 0, System.nanoTime() ); - assertNotNull(index.parsedDoc().dynamicMappingsUpdate()); - } - // with _id - { - MapperService mapper = createMapperService(IndexVersion.current(), indexSettings, () -> false); - SourceToParse source = new SourceToParse("no-such-tsid", new BytesArray(""" - { - "@timestamp": 1609459200000, - "dim": "6a841a21", - "value": 100 - }"""), XContentType.JSON); - var failure = expectThrows(DocumentParsingException.class, () -> { - IndexShard.prepareIndex( - mapper, - source, - UNASSIGNED_SEQ_NO, - randomNonNegativeLong(), - Versions.MATCH_ANY, - VersionType.INTERNAL, - Engine.Operation.Origin.PRIMARY, - -1, - false, - UNASSIGNED_SEQ_NO, - 0, - System.nanoTime() - ); - }); - assertThat( - failure.getMessage(), - equalTo( - "[5:1] failed to parse: _id must be unset or set to [AAAAAMpxfIC8Wpr0AAABdrs-cAA]" - + " but was [no-such-tsid] because [index] is in time_series mode" - ) + }); + assertThat(failure.getMessage(), equalTo("[5:1] failed to parse: Illegal base64 character 20")); + } + + public void testParseWithDynamicMappingNullId() { + Settings indexSettings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim") + .build(); + MapperService mapper = createMapperService(IndexVersion.current(), indexSettings, () -> false); + SourceToParse source = new SourceToParse(null, new BytesArray(""" + { + "@timestamp": 1609459200000, + "dim": "6a841a21", + "value": 100 + }"""), XContentType.JSON); + var failure = expectThrows(DocumentParsingException.class, () -> { + IndexShard.prepareIndex( + mapper, + source, + UNASSIGNED_SEQ_NO, + randomNonNegativeLong(), + Versions.MATCH_ANY, + VersionType.INTERNAL, + Engine.Operation.Origin.PRIMARY, + -1, + false, + UNASSIGNED_SEQ_NO, + 0, + System.nanoTime() ); - } + }); + assertThat( + failure.getMessage(), + equalTo("[5:1] failed to parse: _ts_routing_hash was null but must be set because index [index] is in time_series mode") + ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java new file mode 100644 index 0000000000000..df5ff9a8fe7e5 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class TimeSeriesRoutingHashFieldMapperTests extends MetadataMapperTestCase { + + @Override + protected String fieldName() { + return TimeSeriesRoutingHashFieldMapper.NAME; + } + + @Override + protected boolean isConfigurable() { + return false; + } + + @Override + protected void registerParameters(ParameterChecker checker) throws IOException { + // There aren't any parameters + } + + private DocumentMapper createMapper(XContentBuilder mappings) throws IOException { + return createMapperService( + getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing path is required") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") + .build(), + mappings + ).documentMapper(); + } + + private static ParsedDocument parseDocument(int hash, DocumentMapper docMapper, CheckedConsumer f) + throws IOException { + // Add the @timestamp field required by DataStreamTimestampFieldMapper for all time series indices + return docMapper.parse(source(null, b -> { + f.accept(b); + b.field("@timestamp", "2021-10-01"); + }, TimeSeriesRoutingHashFieldMapper.encode(hash))); + } + + private static int getRoutingHash(ParsedDocument document) { + BytesRef value = document.rootDoc().getBinaryValue(TimeSeriesRoutingHashFieldMapper.NAME); + return TimeSeriesRoutingHashFieldMapper.decode(Uid.decodeId(value.bytes)); + } + + @SuppressWarnings("unchecked") + public void testEnabledInTimeSeriesMode() throws Exception { + DocumentMapper docMapper = createMapper(mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + + int hash = randomInt(); + ParsedDocument doc = parseDocument(hash, docMapper, b -> b.field("a", "value")); + assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); + assertEquals(hash, getRoutingHash(doc)); + } + + public void testDisabledInStandardMode() throws Exception { + DocumentMapper docMapper = createMapperService( + getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()).build(), + mapping(b -> {}) + ).documentMapper(); + assertThat(docMapper.metadataMapper(TimeSeriesRoutingHashFieldMapper.class), is(nullValue())); + + ParsedDocument doc = docMapper.parse(source("id", b -> b.field("field", "value"), null)); + assertThat(doc.rootDoc().getBinaryValue("_ts_routing_hash"), is(nullValue())); + assertThat(doc.rootDoc().get("field"), equalTo("value")); + } + + public void testIncludeInDocumentNotAllowed() throws Exception { + DocumentMapper docMapper = createMapper(mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + Exception e = expectThrows( + DocumentParsingException.class, + () -> parseDocument(randomInt(), docMapper, b -> b.field("_ts_routing_hash", "foo")) + ); + + assertThat( + e.getCause().getMessage(), + containsString("Field [_ts_routing_hash] is a metadata field and cannot be added inside a document") + ); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index c19c21d54a569..0c176a0302620 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -11,10 +11,10 @@ import org.apache.lucene.index.IndexableField; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.name.Named; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexSettings; @@ -25,12 +25,14 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.stream.Stream; import static org.hamcrest.Matchers.equalTo; public class TsidExtractingIdFieldMapperTests extends MetadataMapperTestCase { + private static class TestCase { private final String name; private final String expectedId; @@ -82,7 +84,7 @@ public static Iterable params() { items.add( new TestCase( "2022-01-01T01:00:00Z", - "XsFI2ajcFfi45iV3AAABfhMmioA", + "BwAAAKjcFfi45iV3AAABfhMmioA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { @@ -94,7 +96,7 @@ public static Iterable params() { items.add( new TestCase( "2022-01-01T01:00:01Z", - "XsFI2ajcFfi45iV3AAABfhMmjmg", + "BwAAAKjcFfi45iV3AAABfhMmjmg", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:01.000Z", b -> { @@ -106,7 +108,7 @@ public static Iterable params() { items.add( new TestCase( "1970-01-01T00:00:00Z", - "XsFI2ajcFfi45iV3AAAAAAAAAAA", + "BwAAAKjcFfi45iV3AAAAAAAAAAA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "1970-01-01T00:00:00.000Z", b -> { @@ -118,7 +120,7 @@ public static Iterable params() { items.add( new TestCase( "-9998-01-01T00:00:00Z", - "XsFI2ajcFfi45iV3__6oggRgGAA", + "BwAAAKjcFfi45iV3__6oggRgGAA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "-9998-01-01T00:00:00.000Z", b -> { @@ -130,7 +132,7 @@ public static Iterable params() { items.add( new TestCase( "9998-01-01T00:00:00Z", - "XsFI2ajcFfi45iV3AADmaSK9hAA", + "BwAAAKjcFfi45iV3AADmaSK9hAA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "9998-01-01T00:00:00.000Z", b -> { @@ -144,7 +146,7 @@ public static Iterable params() { items.add( new TestCase( "r1", - "XsFI2ajcFfi45iV3AAABfhMmioA", + "BwAAAKjcFfi45iV3AAABfhMmioA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { @@ -180,7 +182,7 @@ public static Iterable params() { items.add( new TestCase( "r2", - "1y-UzR0iuE1-sOQpAAABfhMmioA", + "BwAAAB0iuE1-sOQpAAABfhMmioA", "JNY_frTR9GmCbhXgK4Y8W44GlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { @@ -192,7 +194,7 @@ public static Iterable params() { items.add( new TestCase( "o.r3", - "zh4dcS1h1gf2J5a8AAABfhMmioA", + "BwAAAC1h1gf2J5a8AAABfhMmioA", "JEyfZsJIp3UNyfWG-4SjKFIGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { @@ -209,7 +211,7 @@ public static Iterable params() { items.add( new TestCase( "k1=dog", - "XsFI2SrEiVgZlSsYAAABfhMmioA", + "BwAAACrEiVgZlSsYAAABfhMmioA", "KJQKpjU9U63jhh-eNJ1f8bipyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { @@ -222,7 +224,7 @@ public static Iterable params() { items.add( new TestCase( "k1=pumpkin", - "XsFI2W8GX8-0QcFxAAABfhMmioA", + "BwAAAG8GX8-0QcFxAAABfhMmioA", "KJQKpjU9U63jhh-eNJ1f8bibzw1JBpU_-VsHjSz5HC1yy_swPEM1iGo", "2022-01-01T01:00:00.000Z", b -> { @@ -235,7 +237,7 @@ public static Iterable params() { items.add( new TestCase( "k1=empty string", - "XsFI2cna58i6D-Q6AAABfhMmioA", + "BwAAAMna58i6D-Q6AAABfhMmioA", "KJQKpjU9U63jhh-eNJ1f8bhaCD7uBpU_-SWGG0Uv9tZ1mLO2gi9rC1I", "2022-01-01T01:00:00.000Z", b -> { @@ -248,7 +250,7 @@ public static Iterable params() { items.add( new TestCase( "k2", - "XsFI2VqlzAuv-06kAAABfhMmioA", + "BwAAAFqlzAuv-06kAAABfhMmioA", "KB9H-tGrL_UzqMcqXcgBtzypyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { @@ -261,7 +263,7 @@ public static Iterable params() { items.add( new TestCase( "o.k3", - "XsFI2S_VhridAKDUAAABfhMmioA", + "BwAAAC_VhridAKDUAAABfhMmioA", "KGXATwN7ISd1_EycFRJ9h6qpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { @@ -274,7 +276,7 @@ public static Iterable params() { items.add( new TestCase( "o.r3", - "zh4dcUwfL7x__2oPAAABfhMmioA", + "BwAAAEwfL7x__2oPAAABfhMmioA", "KJaYZVZz8plfkEvvPBpi1EWpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { @@ -305,7 +307,7 @@ public static Iterable params() { items.add( new TestCase( "L1=1", - "XsFI2fIe53BtV9PCAAABfhMmioA", + "BwAAAPIe53BtV9PCAAABfhMmioA", "KI4kVxcCLIMM2_VQGD575d-tm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "2022-01-01T01:00:00.000Z", b -> { @@ -318,7 +320,7 @@ public static Iterable params() { items.add( new TestCase( "L1=min", - "XsFI2Qhu7hy1RoXRAAABfhMmioA", + "BwAAAAhu7hy1RoXRAAABfhMmioA", "KI4kVxcCLIMM2_VQGD575d8caJ3TBpU_-cLpg-VnCBnhYk33HZBle6E", "2022-01-01T01:00:00.000Z", b -> { @@ -331,7 +333,7 @@ public static Iterable params() { items.add( new TestCase( "L2=1234", - "XsFI2QTrNu7TTpc-AAABfhMmioA", + "BwAAAATrNu7TTpc-AAABfhMmioA", "KI_1WxF60L0IczG5ftUCWdndcGtgBpU_-QfM2BaR0DMagIfw3TDu_mA", "2022-01-01T01:00:00.000Z", b -> { @@ -344,7 +346,7 @@ public static Iterable params() { items.add( new TestCase( "o.L3=max", - "zh4dcWBQI6THHqxoAAABfhMmioA", + "BwAAAGBQI6THHqxoAAABfhMmioA", "KN4a6QzKhzc3nwzNLuZkV51xxTOVBpU_-erUU1qSW4eJ0kP0RmAB9TE", "2022-01-01T01:00:00.000Z", b -> { @@ -375,7 +377,7 @@ public static Iterable params() { items.add( new TestCase( "i1=1", - "XsFI2UMS_RWRoHYjAAABfhMmioA", + "BwAAAEMS_RWRoHYjAAABfhMmioA", "KLGFpvAV8QkWSmX54kXFMgitm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "2022-01-01T01:00:00.000Z", b -> { @@ -388,7 +390,7 @@ public static Iterable params() { items.add( new TestCase( "i1=min", - "XsFI2adlQM5ILoA1AAABfhMmioA", + "BwAAAKdlQM5ILoA1AAABfhMmioA", "KLGFpvAV8QkWSmX54kXFMgjV8hFQBpU_-WG2MicRGWwJdBKWq2F4qy4", "2022-01-01T01:00:00.000Z", b -> { @@ -401,7 +403,7 @@ public static Iterable params() { items.add( new TestCase( "i2=1234", - "XsFI2bhxfB6J0kBFAAABfhMmioA", + "BwAAALhxfB6J0kBFAAABfhMmioA", "KJc4-5eN1uAlYuAknQQLUlxavn2sBpU_-UEXBjgaH1uYcbayrOhdgpc", "2022-01-01T01:00:00.000Z", b -> { @@ -414,7 +416,7 @@ public static Iterable params() { items.add( new TestCase( "o.i3=max", - "zh4dcelxKf19CbfdAAABfhMmioA", + "BwAAAOlxKf19CbfdAAABfhMmioA", "KKqnzPNBe8ObksSo8rNaIFPZPCcBBpU_-Rhd_U6Jn2pjQz2zpmBuJb4", "2022-01-01T01:00:00.000Z", b -> { @@ -445,7 +447,7 @@ public static Iterable params() { items.add( new TestCase( "s1=1", - "XsFI2Y_y-8kD_BFeAAABfhMmioA", + "BwAAAI_y-8kD_BFeAAABfhMmioA", "KFi_JDbvzWyAawmh8IEXedwGlT_5rZuNb-1ruHTTZhtsXRZpZRwWFoc", "2022-01-01T01:00:00.000Z", b -> { @@ -458,7 +460,7 @@ public static Iterable params() { items.add( new TestCase( "s1=min", - "XsFI2WV8VNVnmPVNAAABfhMmioA", + "BwAAAGV8VNVnmPVNAAABfhMmioA", "KFi_JDbvzWyAawmh8IEXedwGlT_5JgBZj9BSCms2_jgeFFhsmDlNFdM", "2022-01-01T01:00:00.000Z", b -> { @@ -471,7 +473,7 @@ public static Iterable params() { items.add( new TestCase( "s2=1234", - "XsFI2VO8mUr-J5CpAAABfhMmioA", + "BwAAAFO8mUr-J5CpAAABfhMmioA", "KKEQ2p3CkpMH61hNk_SuvI0GlT_53XBrYP5TPdmCR-vREPnt20e9f9w", "2022-01-01T01:00:00.000Z", b -> { @@ -484,7 +486,7 @@ public static Iterable params() { items.add( new TestCase( "o.s3=max", - "zh4dcQKh6K11zWeuAAABfhMmioA", + "BwAAAAKh6K11zWeuAAABfhMmioA", "KKVMoT_-GS95fvIBtR7XK9oGlT_5Dme9-H3sen0WZ7leJpCj7-vXau4", "2022-01-01T01:00:00.000Z", b -> { @@ -515,7 +517,7 @@ public static Iterable params() { items.add( new TestCase( "b1=1", - "XsFI2dKxqgT5JDQfAAABfhMmioA", + "BwAAANKxqgT5JDQfAAABfhMmioA", "KGPAUhTjWOsRfDmYp3SUELatm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "2022-01-01T01:00:00.000Z", b -> { @@ -528,7 +530,7 @@ public static Iterable params() { items.add( new TestCase( "b1=min", - "XsFI2d_PD--DgUvoAAABfhMmioA", + "BwAAAN_PD--DgUvoAAABfhMmioA", "KGPAUhTjWOsRfDmYp3SUELYoK6qHBpU_-d8HkZFJ3aL2ZV1lgHAjT1g", "2022-01-01T01:00:00.000Z", b -> { @@ -541,7 +543,7 @@ public static Iterable params() { items.add( new TestCase( "b2=12", - "XsFI2aqX5QjiuhsEAAABfhMmioA", + "BwAAAKqX5QjiuhsEAAABfhMmioA", "KA58oUMzXeX1V5rh51Ste0K5K9vPBpU_-Wn8JQplO-x3CgoslYO5Vks", "2022-01-01T01:00:00.000Z", b -> { @@ -554,7 +556,7 @@ public static Iterable params() { items.add( new TestCase( "o.s3=max", - "zh4dccJ4YtN_21XHAAABfhMmioA", + "BwAAAMJ4YtN_21XHAAABfhMmioA", "KIwZH-StJBobjk9tCV-0OgjKmuwGBpU_-Sd-SdnoH3sbfKLgse-briE", "2022-01-01T01:00:00.000Z", b -> { @@ -585,7 +587,7 @@ public static Iterable params() { items.add( new TestCase( "ip1=192.168.0.1", - "XsFI2T5km9raIz_rAAABfhMmioA", + "BwAAAD5km9raIz_rAAABfhMmioA", "KNj6cLPRNEkqdjfOPIbg0wULrOlWBpU_-efWDsz6B6AnnwbZ7GeeocE", "2022-01-01T01:00:00.000Z", b -> { @@ -602,7 +604,7 @@ public static Iterable params() { items.add( new TestCase( "ip1=12.12.45.254", - "XsFI2QWfEH_e_6wIAAABfhMmioA", + "BwAAAAWfEH_e_6wIAAABfhMmioA", "KNj6cLPRNEkqdjfOPIbg0wVhJ08TBpU_-bANzLhvKPczlle7Pq0z8Qw", "2022-01-01T01:00:00.000Z", b -> { @@ -619,7 +621,7 @@ public static Iterable params() { items.add( new TestCase( "ip2=FE80:CD00:0000:0CDE:1257:0000:211E:729C", - "XsFI2WrrLHr1O4iQAAABfhMmioA", + "BwAAAGrrLHr1O4iQAAABfhMmioA", "KNDo3zGxO9HfN9XYJwKw2Z20h-WsBpU_-f4dSOLGSRlL1hoY2mgERuo", "2022-01-01T01:00:00.000Z", b -> { @@ -632,7 +634,7 @@ public static Iterable params() { items.add( new TestCase( "o.ip3=2001:db8:85a3:8d3:1319:8a2e:370:7348", - "zh4dca7d-9aKOS1MAAABfhMmioA", + "BwAAAK7d-9aKOS1MAAABfhMmioA", "KLXDcBBWJAjgJvjSdF_EJwraAQUzBpU_-ba6HZsIyKnGcbmc3KRLlmI", "2022-01-01T01:00:00.000Z", b -> { @@ -663,7 +665,7 @@ public static Iterable params() { items.add( new TestCase( "huge", - "WZKJR_dECvXBSl3xAAABfhMmioA", + "BwAAAPdECvXBSl3xAAABfhMmioA", "LIe18i0rRU_Bt9vB82F46LaS9mrUkvZq1K_2Gi7UEFMhFwNXrLA_H8TLpUr4", "2022-01-01T01:00:00.000Z", b -> { @@ -680,66 +682,50 @@ public static Iterable params() { private final TestCase testCase; + private static final int ROUTING_HASH = 7; + public TsidExtractingIdFieldMapperTests(@Named("testCase") TestCase testCase) { this.testCase = testCase; } public void testExpectedId() throws IOException { - assertThat(parse(null, mapperService(), testCase.source).id(), equalTo(testCase.expectedId)); + assertThat(parse(mapperService(), testCase.source).id(), equalTo(testCase.expectedId)); } public void testProvideExpectedId() throws IOException { assertThat(parse(testCase.expectedId, mapperService(), testCase.source).id(), equalTo(testCase.expectedId)); } - public void testProvideWrongId() { - String wrongId = testCase.expectedId + "wrong"; - Exception e = expectThrows(DocumentParsingException.class, () -> parse(wrongId, mapperService(), testCase.source)); - assertThat( - e.getCause().getMessage(), - equalTo( - "_id must be unset or set to [" - + testCase.expectedId - + "] but was [" - + testCase.expectedId - + "wrong] because [index] is in time_series mode" - ) - ); - } - public void testEquivalentSources() throws IOException { MapperService mapperService = mapperService(); for (CheckedConsumer equivalent : testCase.equivalentSources) { - assertThat(parse(null, mapperService, equivalent).id(), equalTo(testCase.expectedId)); + assertThat(parse(mapperService, equivalent).id(), equalTo(testCase.expectedId)); } } + private ParsedDocument parse(MapperService mapperService, CheckedConsumer source) throws IOException { + return parse(null, mapperService, source); + } + private ParsedDocument parse(@Nullable String id, MapperService mapperService, CheckedConsumer source) throws IOException { try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) { builder.startObject(); source.accept(builder); builder.endObject(); - SourceToParse sourceToParse = new SourceToParse(id, BytesReference.bytes(builder), builder.contentType()); + SourceToParse sourceToParse = new SourceToParse( + id, + BytesReference.bytes(builder), + builder.contentType(), + TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH) + ); return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup()); } } public void testRoutingPathCompliant() throws IOException { - IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random()); - IndexRouting indexRouting = createIndexSettings(version, indexSettings(version)).getIndexRouting(); - int indexShard = indexShard(indexRouting); - assertThat(indexRouting.getShard(testCase.expectedId, null), equalTo(indexShard)); - assertThat(indexRouting.deleteShard(testCase.expectedId, null), equalTo(indexShard)); - } - - private int indexShard(IndexRouting indexRouting) throws IOException { - try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) { - builder.startObject(); - testCase.source.accept(builder); - builder.endObject(); - return indexRouting.indexShard(null, null, builder.contentType(), BytesReference.bytes(builder)); - } + byte[] bytes = Base64.getUrlDecoder().decode(testCase.expectedId); + assertEquals(ROUTING_HASH, ByteUtils.readIntLE(bytes, 0)); } private Settings indexSettings(IndexVersion version) { @@ -800,7 +786,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { public void testSourceDescription() throws IOException { assertThat(TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext()), equalTo("a time series document")); - ParsedDocument d = parse(null, mapperService(), testCase.randomSource()); + ParsedDocument d = parse(mapperService(), testCase.randomSource()); IndexableField timestamp = d.rootDoc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); assertThat( TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(timestamp)), @@ -830,7 +816,7 @@ private TestDocumentParserContext documentParserContext(IndexableField... fields public void testParsedDescription() throws IOException { assertThat( - TsidExtractingIdFieldMapper.INSTANCE.documentDescription(parse(null, mapperService(), testCase.randomSource())), + TsidExtractingIdFieldMapper.INSTANCE.documentDescription(parse(mapperService(), testCase.randomSource())), equalTo("[" + testCase.expectedId + "][" + testCase.expectedTsid + "@" + testCase.expectedTimestamp + "]") ); } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index 1648e38a3f0b9..cade1e66c7fc7 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESTestCase; @@ -80,6 +81,7 @@ public Map getMetadataMappers() { IdFieldMapper.NAME, RoutingFieldMapper.NAME, TimeSeriesIdFieldMapper.NAME, + TimeSeriesRoutingHashFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, NestedPathFieldMapper.NAME, diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 82fb694db6c66..86c111d1c7145 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -56,6 +56,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.index.seqno.RetentionLease; import org.elasticsearch.index.seqno.RetentionLeases; @@ -521,7 +522,7 @@ public Engine.Index createIndexOp(int docIdent) { { "@timestamp": %s, "dim": "dim" - }""", docIdent)), XContentType.JSON); + }""", docIdent)), XContentType.JSON, TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE); return IndexShard.prepareIndex( mapper, source, diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index 423990999fabd..d6e33c43e94c5 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -22,12 +22,10 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; import java.io.IOException; -import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import static java.util.Collections.emptyList; @@ -89,14 +87,7 @@ public Engine.Operation convertToEngineOp(Translog.Operation operation, Engine.O final Translog.Index index = (Translog.Index) operation; final Engine.Index engineIndex = IndexShard.prepareIndex( mapperService, - new SourceToParse( - index.id(), - index.source(), - XContentHelper.xContentType(index.source()), - index.routing(), - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE - ), + new SourceToParse(index.id(), index.source(), XContentHelper.xContentType(index.source()), index.routing()), index.seqNo(), index.primaryTerm(), index.version(), diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index c393042f07413..09c6eed08bf28 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -714,18 +714,7 @@ protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer