diff --git a/docs/changelog/143151.yaml b/docs/changelog/143151.yaml new file mode 100644 index 0000000000000..1602047ed7371 --- /dev/null +++ b/docs/changelog/143151.yaml @@ -0,0 +1,5 @@ +area: TSDB +issues: [] +pr: 143151 +summary: Support nested documents in time-series indices with synthetic id +type: enhancement diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java index 02faf44830232..4381b2bfcf611 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.DocWriteRequest; @@ -30,6 +31,7 @@ import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; @@ -196,8 +198,9 @@ public void testInvalidCodec() { public void testSyntheticId() throws Exception { assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); + final boolean useNestedDocs = rarely(); final var dataStreamName = randomIdentifier(); - putDataStreamTemplate(dataStreamName, randomIntBetween(1, 5), 0); + putDataStreamTemplate(dataStreamName, randomIntBetween(1, 5), 0, useNestedDocs); final var docs = new HashMap(); final var unit = randomFrom(ChronoUnit.SECONDS, ChronoUnit.MINUTES); @@ -210,21 +213,21 @@ public void testSyntheticId() throws Exception { var results = createDocuments( dataStreamName, // t + 0s - document(timestamp, "vm-dev01", "cpu-load", 0), - document(timestamp, "vm-dev02", "cpu-load", 1), + document(timestamp, "vm-dev01", "cpu-load", 0, useNestedDocs), + document(timestamp, "vm-dev02", "cpu-load", 1, useNestedDocs), // t + 1s - document(timestamp.plus(1, unit), "vm-dev01", "cpu-load", 2), - document(timestamp.plus(1, unit), "vm-dev02", "cpu-load", 3), + document(timestamp.plus(1, unit), "vm-dev01", "cpu-load", 2, useNestedDocs), + document(timestamp.plus(1, unit), "vm-dev02", "cpu-load", 3, useNestedDocs), // t + 0s out-of-order doc - document(timestamp, "vm-dev03", "cpu-load", 4), + document(timestamp, "vm-dev03", "cpu-load", 4, useNestedDocs), // t + 2s - document(timestamp.plus(2, unit), "vm-dev01", "cpu-load", 5), - document(timestamp.plus(2, unit), "vm-dev02", "cpu-load", 6), + document(timestamp.plus(2, unit), "vm-dev01", "cpu-load", 5, useNestedDocs), + document(timestamp.plus(2, unit), "vm-dev02", "cpu-load", 6, useNestedDocs), // t - 1s out-of-order doc - document(timestamp.minus(1, unit), "vm-dev01", "cpu-load", 7), + document(timestamp.minus(1, unit), "vm-dev01", "cpu-load", 7, useNestedDocs), // t + 3s - document(timestamp.plus(3, unit), "vm-dev01", "cpu-load", 8), - document(timestamp.plus(3, unit), "vm-dev02", "cpu-load", 9) + document(timestamp.plus(3, unit), "vm-dev01", "cpu-load", 8, useNestedDocs), + document(timestamp.plus(3, unit), "vm-dev02", "cpu-load", 9, useNestedDocs) ); // Verify that documents are created @@ -296,7 +299,7 @@ enum Operation { if (--nbDocs < 0) { break; } - arrayOfDocs[nbDocs] = document(t, host, "cpu-load", randomInt(10)); + arrayOfDocs[nbDocs] = document(t, host, "cpu-load", randomInt(10), useNestedDocs); } // always use seconds, otherwise the doc might fell outside of the timestamps window of the datastream t = t.plus(1, ChronoUnit.SECONDS); @@ -327,6 +330,39 @@ enum Operation { } }); + if (useNestedDocs) { + assertCheckedResponse( + client().prepareSearch(dataStreamName) + .setTrackTotalHits(true) + .setQuery(QueryBuilders.nestedQuery("tags", QueryBuilders.existsQuery("tags.key"), ScoreMode.None)), + searchResponse -> { + assertHitCount(searchResponse, docs.size() - deletedDocs.size()); + for (var hit : searchResponse.getHits()) { + assertThat( + "Nested query returned deleted doc [" + hit.getId() + "]", + deletedDocs.contains(hit.getId()), + equalTo(false) + ); + } + } + ); + + for (var deletedDocId : deletedDocs) { + var deletedDocIndex = docs.get(deletedDocId); + assertHitCount( + client().prepareSearch(deletedDocIndex) + .setTrackTotalHits(true) + .setSize(0) + .setQuery( + QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(IdFieldMapper.NAME, deletedDocId)) + .must(QueryBuilders.nestedQuery("tags", QueryBuilders.existsQuery("tags.key"), ScoreMode.None)) + ), + 0L + ); + } + } + // Search by synthetic _id var otherDocs = randomSubsetOf(Sets.difference(docs.keySet(), Sets.newHashSet(deletedDocs))); assertSearchById(otherDocs, docs); @@ -368,21 +404,21 @@ enum Operation { var bulkResponses = createDocumentsWithoutValidatingTheResponse( dataStreamName, // t + 0s - document(timestamp, "vm-dev01", "cpu-load", 0), - document(timestamp, "vm-dev02", "cpu-load", 1), + document(timestamp, "vm-dev01", "cpu-load", 0, useNestedDocs), + document(timestamp, "vm-dev02", "cpu-load", 1, useNestedDocs), // t + 1s - document(timestamp.plus(1, unit), "vm-dev01", "cpu-load", 2), - document(timestamp.plus(1, unit), "vm-dev02", "cpu-load", 3), + document(timestamp.plus(1, unit), "vm-dev01", "cpu-load", 2, useNestedDocs), + document(timestamp.plus(1, unit), "vm-dev02", "cpu-load", 3, useNestedDocs), // t + 0s out-of-order doc - document(timestamp, "vm-dev03", "cpu-load", 4), + document(timestamp, "vm-dev03", "cpu-load", 4, useNestedDocs), // t + 2s - document(timestamp.plus(2, unit), "vm-dev01", "cpu-load", 5), - document(timestamp.plus(2, unit), "vm-dev02", "cpu-load", 6), + document(timestamp.plus(2, unit), "vm-dev01", "cpu-load", 5, useNestedDocs), + document(timestamp.plus(2, unit), "vm-dev02", "cpu-load", 6, useNestedDocs), // t - 1s out-of-order doc - document(timestamp.minus(1, unit), "vm-dev01", "cpu-load", 7), + document(timestamp.minus(1, unit), "vm-dev01", "cpu-load", 7, useNestedDocs), // t + 3s - document(timestamp.plus(3, unit), "vm-dev01", "cpu-load", 8), - document(timestamp.plus(3, unit), "vm-dev02", "cpu-load", 9) + document(timestamp.plus(3, unit), "vm-dev01", "cpu-load", 8, useNestedDocs), + document(timestamp.plus(3, unit), "vm-dev02", "cpu-load", 9, useNestedDocs) ); var successfulRequests = Arrays.stream(bulkResponses).filter(response -> response.isFailed() == false).toList(); @@ -408,8 +444,9 @@ enum Operation { public void testGetFromTranslogBySyntheticId() throws Exception { assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); + final boolean useNestedDocs = rarely(); final var dataStreamName = randomIdentifier(); - putDataStreamTemplate(dataStreamName, 1, 0); + putDataStreamTemplate(dataStreamName, 1, 0, useNestedDocs); final var docs = new HashMap(); final var unit = randomFrom(ChronoUnit.SECONDS, ChronoUnit.MINUTES); @@ -421,13 +458,13 @@ public void testGetFromTranslogBySyntheticId() throws Exception { var results = createDocuments( dataStreamName, // t + 0s - document(timestamp, "vm-dev01", "cpu-load", 0), - document(timestamp, "vm-dev02", "cpu-load", 1), + document(timestamp, "vm-dev01", "cpu-load", 0, useNestedDocs), + document(timestamp, "vm-dev02", "cpu-load", 1, useNestedDocs), // t + 1s - document(timestamp.plus(1, unit), "vm-dev01", "cpu-load", 2), - document(timestamp.plus(1, unit), "vm-dev02", "cpu-load", 3), + document(timestamp.plus(1, unit), "vm-dev01", "cpu-load", 2, useNestedDocs), + document(timestamp.plus(1, unit), "vm-dev02", "cpu-load", 3, useNestedDocs), // t + 0s out-of-order doc - document(timestamp, "vm-dev03", "cpu-load", 4) + document(timestamp, "vm-dev03", "cpu-load", 4, useNestedDocs) ); // Verify that documents are created @@ -459,13 +496,13 @@ public void testGetFromTranslogBySyntheticId() throws Exception { results = createDocuments( dataStreamName, // t + 2s - document(timestamp.plus(2, unit), "vm-dev01", "cpu-load", metricOffset), - document(timestamp.plus(2, unit), "vm-dev02", "cpu-load", metricOffset + 1), + document(timestamp.plus(2, unit), "vm-dev01", "cpu-load", metricOffset, useNestedDocs), + document(timestamp.plus(2, unit), "vm-dev02", "cpu-load", metricOffset + 1, useNestedDocs), // t - 1s out-of-order doc - document(timestamp.minus(1, unit), "vm-dev01", "cpu-load", metricOffset + 2), + document(timestamp.minus(1, unit), "vm-dev01", "cpu-load", metricOffset + 2, useNestedDocs), // t + 3s - document(timestamp.plus(3, unit), "vm-dev01", "cpu-load", metricOffset + 3), - document(timestamp.plus(3, unit), "vm-dev02", "cpu-load", metricOffset + 4) + document(timestamp.plus(3, unit), "vm-dev01", "cpu-load", metricOffset + 3, useNestedDocs), + document(timestamp.plus(3, unit), "vm-dev02", "cpu-load", metricOffset + 4, useNestedDocs) ); // Verify that documents are created @@ -510,6 +547,16 @@ public void testGetFromTranslogBySyntheticId() throws Exception { assertHitCount(client().prepareSearch(dataStreamName).setSize(0), 10L); + if (useNestedDocs) { + assertHitCount( + client().prepareSearch(dataStreamName) + .setTrackTotalHits(true) + .setSize(0) + .setQuery(QueryBuilders.nestedQuery("tags", QueryBuilders.existsQuery("tags.key"), ScoreMode.None)), + 10L + ); + } + // Check that synthetic _id field have no postings on disk but has bloom filter usage var indices = new HashSet<>(docs.values()); for (var index : indices) { @@ -524,13 +571,14 @@ public void testGetFromTranslogBySyntheticId() throws Exception { public void testRecoveredOperations() throws Exception { assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); + final boolean useNestedDocs = rarely(); // ensure a couple of nodes to have some operations coordinated internalCluster().ensureAtLeastNumDataNodes(2); final var dataStreamName = randomIdentifier(); final int numShards = randomIntBetween(1, 10); - putDataStreamTemplate(dataStreamName, numShards, 0); + putDataStreamTemplate(dataStreamName, numShards, 0, useNestedDocs); final var docsIndices = new HashSet(); final var docsIndicesById = new HashMap(); @@ -547,7 +595,7 @@ public void testRecoveredOperations() throws Exception { var client = client(); var bulkRequest = client.prepareBulk(); for (int j = 0; j < nbDocsPerBulk; j++) { - var doc = document(timestamp, randomFrom("vm-dev01", "vm-dev02", "vm-dev03", "vm-dev04"), "cpu-load", i); + var doc = document(timestamp, randomFrom("vm-dev01", "vm-dev02", "vm-dev03", "vm-dev04"), "cpu-load", i, useNestedDocs); bulkRequest.add(client.prepareIndex(dataStreamName).setOpType(DocWriteRequest.OpType.CREATE).setSource(doc)); timestamp = timestamp.plusMillis(1); } @@ -605,7 +653,8 @@ public void testRecoveredOperations() throws Exception { indexShard.mapperService().documentMapper(), operation, docsIdsBySeqNo::get, - docsIndicesById::get + docsIndicesById::get, + useNestedDocs ); } } @@ -632,7 +681,8 @@ public void testRecoveredOperations() throws Exception { indexShard.mapperService().documentMapper(), operation, docsIdsBySeqNo::get, - docsIndicesById::get + docsIndicesById::get, + useNestedDocs ); } } @@ -708,6 +758,16 @@ enum Operation { final var nonDeletedDocs = Sets.difference(docsIndicesById.keySet(), Set.copyOf(deletedDocs)); assertHitCount(client(targetNode).prepareSearch(dataStreamName).setTrackTotalHits(true).setSize(0), nonDeletedDocs.size()); + if (useNestedDocs) { + assertHitCount( + client(targetNode).prepareSearch(dataStreamName) + .setTrackTotalHits(true) + .setSize(0) + .setQuery(QueryBuilders.nestedQuery("tags", QueryBuilders.existsQuery("tags.key"), ScoreMode.None)), + nonDeletedDocs.size() + ); + } + var randomDocIds = randomSubsetOf(nonDeletedDocs); for (var docId : randomDocIds) { if (randomBoolean()) { @@ -756,6 +816,7 @@ enum Operation { public void testRecoverOperationsFromLocalTranslog() throws Exception { assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); + final boolean useNestedDocs = rarely(); final var dataStreamName = randomIdentifier(); putDataStreamTemplate( @@ -765,7 +826,8 @@ public void testRecoverOperationsFromLocalTranslog() throws Exception { Settings.builder() .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST) .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), ByteSizeValue.of(1, ByteSizeUnit.PB)) - .build() + .build(), + useNestedDocs ); final var docsIndices = new HashSet(); @@ -781,7 +843,7 @@ public void testRecoverOperationsFromLocalTranslog() throws Exception { var client = client(); var bulkRequest = client.prepareBulk(); for (int i = 0; i < nbDocs; i++) { - var doc = document(timestamp, randomFrom("vm-dev01", "vm-dev02", "vm-dev03", "vm-dev04"), "cpu-load", i); + var doc = document(timestamp, randomFrom("vm-dev01", "vm-dev02", "vm-dev03", "vm-dev04"), "cpu-load", i, useNestedDocs); bulkRequest.add(client.prepareIndex(dataStreamName).setOpType(DocWriteRequest.OpType.CREATE).setSource(doc)); timestamp = timestamp.plusMillis(1); } @@ -853,7 +915,8 @@ public void testRecoverOperationsFromLocalTranslog() throws Exception { primary.mapperService().documentMapper(), operation, docsIdsBySeqNo::get, - docsIndicesById::get + docsIndicesById::get, + useNestedDocs ); } } @@ -886,6 +949,16 @@ public void testRecoverOperationsFromLocalTranslog() throws Exception { final var nonDeletedDocs = Sets.difference(docsIndicesById.keySet(), Set.copyOf(deletedDocs)); assertHitCount(client().prepareSearch(dataStreamName).setTrackTotalHits(true).setSize(0), nonDeletedDocs.size()); + if (useNestedDocs) { + assertHitCount( + client().prepareSearch(dataStreamName) + .setTrackTotalHits(true) + .setSize(0) + .setQuery(QueryBuilders.nestedQuery("tags", QueryBuilders.existsQuery("tags.key"), ScoreMode.None)), + nonDeletedDocs.size() + ); + } + for (var docId : randomSubsetOf(nonDeletedDocs)) { if (randomBoolean()) { var getResponse = client().prepareGet(docsIndicesById.get(docId), docId) @@ -935,7 +1008,8 @@ private static void assertTranslogOperation( DocumentMapper documentMapper, Translog.Operation operation, Function expectedDocIdSupplier, - Function expectedDocIndexSupplier + Function expectedDocIndexSupplier, + boolean useNestedDocs ) { final String expectedDocId; final BytesRef expectedDocIdEncoded; @@ -963,9 +1037,13 @@ private static void assertTranslogOperation( ); assertThat(parsedDocument.id(), equalTo(expectedDocId)); assertThat(parsedDocument.routing(), nullValue()); - assertThat(parsedDocument.docs(), hasSize(1)); + if (useNestedDocs) { + assertThat(parsedDocument.docs(), hasSize(greaterThan(1))); + } else { + assertThat(parsedDocument.docs(), hasSize(1)); + } - var luceneDocument = parsedDocument.docs().get(0); + var luceneDocument = parsedDocument.rootDoc(); assertThat( "Lucene document [" + expectedDocId + "] has wrong value for _id field", luceneDocument.getField(IdFieldMapper.NAME).binaryValue(), @@ -992,6 +1070,36 @@ private static void assertTranslogOperation( ) ) ); + + for (int i = 0; i < parsedDocument.docs().size() - 1; i++) { + var nestedDoc = parsedDocument.docs().get(i); + assertThat( + "Nested document [" + i + "] of [" + expectedDocId + "] has wrong _id field", + nestedDoc.getField(IdFieldMapper.NAME).binaryValue(), + equalTo(expectedDocIdEncoded) + ); + assertThat( + "Nested document [" + i + "] of [" + expectedDocId + "] has wrong _tsid field", + nestedDoc.getField(TimeSeriesIdFieldMapper.NAME).binaryValue(), + equalTo(TsidExtractingIdFieldMapper.extractTimeSeriesIdFromSyntheticId(expectedDocIdEncoded)) + ); + assertThat( + "Nested document [" + i + "] of [" + expectedDocId + "] has wrong @timestamp field", + nestedDoc.getField(DataStreamTimestampFieldMapper.DEFAULT_PATH).numericValue().longValue(), + equalTo(TsidExtractingIdFieldMapper.extractTimestampFromSyntheticId(expectedDocIdEncoded)) + ); + assertThat( + "Nested document [" + i + "] of [" + expectedDocId + "] has wrong _ts_routing_hash field", + nestedDoc.getField(TimeSeriesRoutingHashFieldMapper.NAME).binaryValue(), + equalTo( + Uid.encodeId( + TimeSeriesRoutingHashFieldMapper.encode( + TsidExtractingIdFieldMapper.extractRoutingHashFromSyntheticId(expectedDocIdEncoded) + ) + ) + ) + ); + } break; case DELETE: @@ -1016,11 +1124,12 @@ private static void assertTranslogOperation( */ public void testCreateSnapshot() throws IOException { assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); + final boolean useNestedDocs = rarely(); // create index final var dataStreamName = randomIdentifier(); int shards = randomIntBetween(1, 5); - putDataStreamTemplate(dataStreamName, shards, 0); + putDataStreamTemplate(dataStreamName, shards, 0, useNestedDocs); final var unit = randomFrom(ChronoUnit.SECONDS, ChronoUnit.MINUTES); final var timestamp = Instant.now(); @@ -1029,21 +1138,21 @@ public void testCreateSnapshot() throws IOException { var bulkItemResponses = createDocuments( dataStreamName, // t + 0s - document(timestamp, "vm-dev01", "cpu-load", 0), - document(timestamp, "vm-dev02", "cpu-load", 1), + document(timestamp, "vm-dev01", "cpu-load", 0, useNestedDocs), + document(timestamp, "vm-dev02", "cpu-load", 1, useNestedDocs), // t + 1s - document(timestamp.plus(1, unit), "vm-dev01", "cpu-load", 2), - document(timestamp.plus(1, unit), "vm-dev02", "cpu-load", 3), + document(timestamp.plus(1, unit), "vm-dev01", "cpu-load", 2, useNestedDocs), + document(timestamp.plus(1, unit), "vm-dev02", "cpu-load", 3, useNestedDocs), // t + 0s out-of-order doc - document(timestamp, "vm-dev03", "cpu-load", 4), + document(timestamp, "vm-dev03", "cpu-load", 4, useNestedDocs), // t + 2s - document(timestamp.plus(2, unit), "vm-dev01", "cpu-load", 5), - document(timestamp.plus(2, unit), "vm-dev02", "cpu-load", 6), + document(timestamp.plus(2, unit), "vm-dev01", "cpu-load", 5, useNestedDocs), + document(timestamp.plus(2, unit), "vm-dev02", "cpu-load", 6, useNestedDocs), // t - 1s out-of-order doc - document(timestamp.minus(1, unit), "vm-dev01", "cpu-load", 7), + document(timestamp.minus(1, unit), "vm-dev01", "cpu-load", 7, useNestedDocs), // t + 3s - document(timestamp.plus(3, unit), "vm-dev01", "cpu-load", 8), - document(timestamp.plus(3, unit), "vm-dev02", "cpu-load", 9) + document(timestamp.plus(3, unit), "vm-dev01", "cpu-load", 8, useNestedDocs), + document(timestamp.plus(3, unit), "vm-dev02", "cpu-load", 9, useNestedDocs) ); // Verify that documents are created @@ -1121,13 +1230,23 @@ public void testCreateSnapshot() throws IOException { // All documents should be there Map> documentSourcesAfterRestore = documentSourcesAsMaps(dataStreamName, docsToVerify); assertThat(documentSourcesAfterRestore, equalTo(documentSourcesBeforeSnapshot)); + + if (useNestedDocs) { + assertHitCount( + client().prepareSearch(dataStreamName) + .setTrackTotalHits(true) + .setSize(0) + .setQuery(QueryBuilders.nestedQuery("tags", QueryBuilders.existsQuery("tags.key"), ScoreMode.None)), + docIdToIndex.size() + ); + } } public void testMerge() throws Exception { assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); final var dataStreamName = randomIdentifier(); - putDataStreamTemplate(dataStreamName, 1, 0); + putDataStreamTemplate(dataStreamName, 1, 0, rarely()); final var docsIndexByIds = new ConcurrentHashMap(); var timestamp = Instant.now(); @@ -1233,6 +1352,16 @@ private static void assertSearchById(Collection searchIds, HashMap indi @Override public Status needsField(FieldInfo fieldInfo) throws IOException { - return IdFieldMapper.NAME.equals(fieldInfo.getName()) ? Status.YES : Status.NO; + return IdFieldMapper.NAME.equals(fieldInfo.getName()) + ? StoredFieldVisitor.Status.YES + : Status.NO; } @Override diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml index f4aca5ab264e8..854a863e40b38 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml @@ -3,6 +3,10 @@ setup: cluster_features: ["mapper.tsdb_nested_field_support"] reason: "tsdb index with nested field support enabled" + - skip: + cluster_features: "index.time_series_synthetic_id" + reason: when cluster has synthetic_id feature use 161/162_nested_fields_synthetic_id_false/true instead + --- "Create TSDB index with field of nested type": - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/161_nested_fields_synthetic_id_false.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/161_nested_fields_synthetic_id_false.yml new file mode 100644 index 0000000000000..ddf591a49b4c0 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/161_nested_fields_synthetic_id_false.yml @@ -0,0 +1,240 @@ +setup: + - requires: + cluster_features: ["mapper.tsdb_nested_field_support"] + reason: "tsdb index with nested field support enabled" + - requires: + cluster_features: ["index.time_series_synthetic_id"] + reason: "index.mapping.synthetic_id requires support for synthetic id in the cluster" + +--- +"Create TSDB index with field of nested type and synthetic_id false": + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 1 + number_of_shards: 1 + routing_path: [department] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mapping: + synthetic_id: false + mappings: + properties: + "@timestamp": + type: date + department: + type: keyword + time_series_dimension: true + staff: + type: integer + courses: + type: nested + properties: + name: + type: keyword + credits: + type: integer + + - do: + index: + index: test + body: { "@timestamp": "2021-04-28T01:00:00Z", "department": "compsci", "staff": 12, "courses": [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4 } ] } + + - do: + index: + index: test + body: { "@timestamp": "2021-04-28T02:00:00Z", "department": "math", "staff": 20, "courses": [ { "name": "Precalculus", "credits": 1 }, { "name": "Linear Algebra", "credits": 3 } ] } + + - do: + indices.refresh: + index: [ test ] + + - do: + search: + index: test + body: + size: 0 + query: + nested: + path: "courses" + query: + bool: + must: + - term: + courses.name: Precalculus + - term: + courses.credits: 3 + + - match: { hits.total.value: 0 } + + - do: + search: + index: test + body: + query: + nested: + path: "courses" + query: + bool: + must: + - term: + courses.name: "Object Oriented Programming" + - term: + courses.credits: 3 + + - match: { hits.total.value: 1 } + - match: { "hits.hits.0._source.@timestamp": "2021-04-28T01:00:00.000Z" } + - match: { hits.hits.0._source.department: "compsci" } + - match: { hits.hits.0._source.courses: [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4, } ] } + +--- + +"TSDB index with multi-level nested fields and synthetic_id false": + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 1 + number_of_shards: 1 + routing_path: [department] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mapping: + synthetic_id: false + mappings: + properties: + "@timestamp": + type: date + department: + type: keyword + time_series_dimension: true + staff: + type: integer + courses: + type: nested + properties: + name: + type: keyword + credits: + type: integer + students: + type: nested + properties: + name: + type: text + major: + type: keyword + + - do: + index: + index: test + body: + "@timestamp": "2021-04-28T01:00:00Z" + department: "compsci" + staff: 12 + courses: + - name: "Object Oriented Programming" + credits: 3 + students: + - name: "Kimora Tanner" + major: "Computer Science" + - name: "Bruno Garrett" + major: "Software Engineering" + - name: "Theory of Computation" + credits: 4 + students: + - name: "Elliott Booker" + major: "Computer Engineering" + - name: "Kimora Tanner" + major: "Software Engineering" + + - do: + index: + index: test + body: + "@timestamp": "2021-04-28T02:00:00Z" + department: "math" + staff: 20 + courses: + - name: "Precalculus" + credits: 4 + students: + - name: "Elliott Ayers" + major: "Software Engineering" + - name: "Sylvie Howe" + major: "Computer Engineering" + - name: "Linear Algebra" + credits: 3 + students: + - name: "Kimora Tanner" + major: "Computer Science" + - name: "Bruno Garett" + major: "Software Engineering" + - name: "Amelia Booker" + major: "Psychology" + + - do: + index: + index: test + body: + "@timestamp": "2021-04-28T03:00:00Z" + department: "compsci" + staff: 12 + courses: + - name: "Object Oriented Programming" + credits: 3 + students: + - name: "Kimora Tanner" + major: "Computer Science" + - name: "Bruno Garrett" + major: "Software Engineering" + - name: "Elliott Booker" + major: "Computer Engineering" + - name: "Theory of Computation" + credits: 4 + students: + - name: "Kimora Tanner" + major: "Software Engineering" + - name: "Elliott Ayers" + major: "Software Engineering" + - name: "Apollo Pittman" + major: "Computer Engineering" + + - do: + indices.refresh: + index: [ test ] + + - do: + search: + index: test + body: + query: + nested: + path: "courses" + query: + bool: + must: + - nested: + path: "courses.students" + query: + bool: + must: + - match: + courses.students.name: "Elliott" + - term: + courses.students.major: "Computer Engineering" + - term: + courses.name: "Theory of Computation" + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._source.department: "compsci" } + - match: { "hits.hits.0._source.@timestamp": "2021-04-28T01:00:00.000Z" } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/162_nested_fields_synthetic_id_true.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/162_nested_fields_synthetic_id_true.yml new file mode 100644 index 0000000000000..d25b42d3940b3 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/162_nested_fields_synthetic_id_true.yml @@ -0,0 +1,240 @@ +setup: + - requires: + cluster_features: ["mapper.tsdb_nested_field_support"] + reason: "tsdb index with nested field support enabled" + - requires: + cluster_features: ["index.time_series_synthetic_id"] + reason: "index.mapping.synthetic_id requires support for synthetic id in the cluster" + +--- +"Create TSDB index with field of nested type and synthetic_id true": + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 1 + number_of_shards: 1 + routing_path: [department] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mapping: + synthetic_id: true + mappings: + properties: + "@timestamp": + type: date + department: + type: keyword + time_series_dimension: true + staff: + type: integer + courses: + type: nested + properties: + name: + type: keyword + credits: + type: integer + + - do: + index: + index: test + body: { "@timestamp": "2021-04-28T01:00:00Z", "department": "compsci", "staff": 12, "courses": [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4 } ] } + + - do: + index: + index: test + body: { "@timestamp": "2021-04-28T02:00:00Z", "department": "math", "staff": 20, "courses": [ { "name": "Precalculus", "credits": 1 }, { "name": "Linear Algebra", "credits": 3 } ] } + + - do: + indices.refresh: + index: [ test ] + + - do: + search: + index: test + body: + size: 0 + query: + nested: + path: "courses" + query: + bool: + must: + - term: + courses.name: Precalculus + - term: + courses.credits: 3 + + - match: { hits.total.value: 0 } + + - do: + search: + index: test + body: + query: + nested: + path: "courses" + query: + bool: + must: + - term: + courses.name: "Object Oriented Programming" + - term: + courses.credits: 3 + + - match: { hits.total.value: 1 } + - match: { "hits.hits.0._source.@timestamp": "2021-04-28T01:00:00.000Z" } + - match: { hits.hits.0._source.department: "compsci" } + - match: { hits.hits.0._source.courses: [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4, } ] } + +--- + +"TSDB index with multi-level nested fields and synthetic_id true": + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 1 + number_of_shards: 1 + routing_path: [department] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mapping: + synthetic_id: true + mappings: + properties: + "@timestamp": + type: date + department: + type: keyword + time_series_dimension: true + staff: + type: integer + courses: + type: nested + properties: + name: + type: keyword + credits: + type: integer + students: + type: nested + properties: + name: + type: text + major: + type: keyword + + - do: + index: + index: test + body: + "@timestamp": "2021-04-28T01:00:00Z" + department: "compsci" + staff: 12 + courses: + - name: "Object Oriented Programming" + credits: 3 + students: + - name: "Kimora Tanner" + major: "Computer Science" + - name: "Bruno Garrett" + major: "Software Engineering" + - name: "Theory of Computation" + credits: 4 + students: + - name: "Elliott Booker" + major: "Computer Engineering" + - name: "Kimora Tanner" + major: "Software Engineering" + + - do: + index: + index: test + body: + "@timestamp": "2021-04-28T02:00:00Z" + department: "math" + staff: 20 + courses: + - name: "Precalculus" + credits: 4 + students: + - name: "Elliott Ayers" + major: "Software Engineering" + - name: "Sylvie Howe" + major: "Computer Engineering" + - name: "Linear Algebra" + credits: 3 + students: + - name: "Kimora Tanner" + major: "Computer Science" + - name: "Bruno Garett" + major: "Software Engineering" + - name: "Amelia Booker" + major: "Psychology" + + - do: + index: + index: test + body: + "@timestamp": "2021-04-28T03:00:00Z" + department: "compsci" + staff: 12 + courses: + - name: "Object Oriented Programming" + credits: 3 + students: + - name: "Kimora Tanner" + major: "Computer Science" + - name: "Bruno Garrett" + major: "Software Engineering" + - name: "Elliott Booker" + major: "Computer Engineering" + - name: "Theory of Computation" + credits: 4 + students: + - name: "Kimora Tanner" + major: "Software Engineering" + - name: "Elliott Ayers" + major: "Software Engineering" + - name: "Apollo Pittman" + major: "Computer Engineering" + + - do: + indices.refresh: + index: [ test ] + + - do: + search: + index: test + body: + query: + nested: + path: "courses" + query: + bool: + must: + - nested: + path: "courses.students" + query: + bool: + must: + - match: + courses.students.name: "Elliott" + - term: + courses.students.major: "Computer Engineering" + - term: + courses.name: "Theory of Computation" + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._source.department: "compsci" } + - match: { "hits.hits.0._source.@timestamp": "2021-04-28T01:00:00.000Z" } diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBSyntheticIdDocValuesHolder.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBSyntheticIdDocValuesHolder.java index ce3ea494b8960..7fd86961d2bb2 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBSyntheticIdDocValuesHolder.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBSyntheticIdDocValuesHolder.java @@ -84,7 +84,7 @@ int docTsIdOrdinal(int docID) throws IOException { cachedTsId = null; } boolean found = tsIdDocValues.advanceExact(docID); - assert found : "No value found for field [" + tsIdFieldInfo.getName() + " and docID " + docID; + assert found : "No value found for field [" + tsIdFieldInfo.getName() + "] and docID " + docID; return tsIdDocValues.ordValue(); } @@ -100,7 +100,7 @@ long docTimestamp(int docID) throws IOException { timestampDocValues = docValuesProducer.getSortedNumeric(timestampFieldInfo); } boolean found = timestampDocValues.advanceExact(docID); - assert found : "No value found for field [" + timestampFieldInfo.getName() + " and docID " + docID; + assert found : "No value found for field [" + timestampFieldInfo.getName() + "] and docID " + docID; assert timestampDocValues.docValueCount() == 1; return timestampDocValues.nextValue(); } @@ -117,7 +117,7 @@ BytesRef docRoutingHash(int docID) throws IOException { routingHashDocValues = docValuesProducer.getSorted(routingHashFieldInfo); } boolean found = routingHashDocValues.advanceExact(docID); - assert found : "No value found for field [" + routingHashFieldInfo.getName() + " and docID " + docID; + assert found : "No value found for field [" + routingHashFieldInfo.getName() + "] and docID " + docID; return routingHashDocValues.lookupOrd(routingHashDocValues.ordValue()); } @@ -218,7 +218,7 @@ int findFirstDocWithTsIdOrdinalEqualOrGreaterThan(int tsIdOrd) throws IOExceptio for (int docID = startDocId; docID != DocIdSetIterator.NO_MORE_DOCS; docID = tsIdDocValues.nextDoc()) { boolean found = tsIdDocValues.advanceExact(docID); - assert found : "No value found for field [" + tsIdFieldInfo.getName() + " and docID " + docID; + assert found : "No value found for field [" + tsIdFieldInfo.getName() + "] and docID " + docID; var ord = tsIdDocValues.ordValue(); if (ord == tsIdOrd || tsIdOrd < ord) { if (ord != cachedTsIdOrd) { @@ -256,7 +256,7 @@ int findFirstDocWithTsIdOrdinalEqualTo(int tsIdOrd) throws IOException { for (int docID = startDocId; docID != DocIdSetIterator.NO_MORE_DOCS; docID = tsIdDocValues.nextDoc()) { boolean found = tsIdDocValues.advanceExact(docID); - assert found : "No value found for field [" + tsIdFieldInfo.getName() + " and docID " + docID; + assert found : "No value found for field [" + tsIdFieldInfo.getName() + "] and docID " + docID; var ord = tsIdDocValues.ordValue(); if (ord == tsIdOrd) { if (ord != cachedTsIdOrd) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 0aa0533579407..1c3e314687336 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -10,7 +10,6 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.time.DateFormatter; @@ -18,6 +17,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.vectors.VectorsFormatProvider; @@ -37,6 +37,8 @@ import java.util.Set; import java.util.function.BiConsumer; +import static org.elasticsearch.index.mapper.IdFieldMapper.standardIdField; + /** * Context used when parsing incoming documents. Holds everything that is needed to parse a document as well as * the lucene data structures and mappings to be dynamically created as the outcome of parsing a document. @@ -815,14 +817,17 @@ public final DocumentParserContext createNestedContext(NestedObjectMapper nested if (idField != null) { // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then // delete it when the root document is deleted too. - doc.add(new StringField(IdFieldMapper.NAME, idField.binaryValue(), Field.Store.NO)); + doc.add(standardIdField(idField.binaryValue(), Field.Store.NO)); } else if (indexSettings().getMode() == IndexMode.TIME_SERIES) { // For time series indices, the _id is generated from the _tsid, which in turn is generated from the values of the configured // routing fields. At this point in document parsing, we can't guarantee that we've parsed all the routing fields yet, so the // parent document's _id is not yet available. // So we just add the child document without the parent _id, then in TimeSeriesIdFieldMapper#postParse we set the _id on all // child documents once we've calculated it. - assert getRoutingFields().equals(RoutingFields.Noop.INSTANCE) == false; + // Time-series index created at or after TSID_CREATED_DURING_ROUTING with non-empty dimensions use ForIndexDimensions routing, + // which causes buildRoutingFields to return Noop.INSTANCE. + assert getRoutingFields().equals(RoutingFields.Noop.INSTANCE) == false + || indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING); } else { throw new IllegalStateException("The root document of a nested document should have an _id field"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 1bf27b3288210..92252d6a9827b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -71,23 +71,31 @@ protected final String contentType() { public abstract String reindexId(String id); /** - * Create a {@link Field} to store the provided {@code _id} that "stores" - * the {@code _id} so it can be fetched easily from the index. + * Create an indexed and stored {@link Field} for the provided {@code _id}. */ public static Field standardIdField(String id) { - return new StringField(NAME, Uid.encodeId(id), Field.Store.YES); + return standardIdField(Uid.encodeId(id), Field.Store.YES); } /** - * Create a {@link Field} corresponding to a synthetic {@code _id} field, which is not indexed but instead resolved at runtime. + * Create an indexed {@link Field} for the provided {@code _id}, optionally stored. + * The id must already be encoded using {@link Uid#encodeId(String)}. + */ + public static Field standardIdField(BytesRef uid, Field.Store stored) { + return new StringField(NAME, uid, stored); + } + + /** + * Create a {@link Field} corresponding to a synthetic {@code _id} field, which is not indexed and not stored but instead computed at + * runtime. */ public static Field syntheticIdField(String id) { return new SyntheticIdField(Uid.encodeId(id)); } /** - * Create a {@link Field} corresponding to a synthetic {@code _id} field, which is not indexed but instead resolved at runtime. The id - * must be already encoded using {@link Uid#encodeId(String)}. + * Create a {@link Field} corresponding to a synthetic {@code _id} field, which is not indexed and not stored but instead resolved at + * runtime. The id must be already encoded using {@link Uid#encodeId(String)}. */ public static Field syntheticIdField(BytesRef uid) { return new SyntheticIdField(uid); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java index b1f65e9c3267b..e87b854605719 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java @@ -111,10 +111,10 @@ public static ParsedDocument deleteTombstone( if (useDocValuesSkipper) { document.add(SortedDocValuesField.indexedField(TimeSeriesIdFieldMapper.NAME, timeSeriesId)); - document.add(SortedNumericDocValuesField.indexedField("@timestamp", timestamp)); + document.add(SortedNumericDocValuesField.indexedField(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp)); } else { document.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, timeSeriesId)); - document.add(new LongField("@timestamp", timestamp, Field.Store.NO)); + document.add(new LongField(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp, Field.Store.NO)); } var field = new SortedDocValuesField( TimeSeriesRoutingHashFieldMapper.NAME, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index d886ca83af9be..46ec236ac9e5c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -10,8 +10,10 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongField; import org.apache.lucene.document.SortedDocValuesField; -import org.apache.lucene.document.StringField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.routing.RoutingHashBuilder; @@ -35,10 +37,14 @@ import java.io.IOException; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.SortedMap; +import static org.elasticsearch.index.mapper.IdFieldMapper.standardIdField; +import static org.elasticsearch.index.mapper.IdFieldMapper.syntheticIdField; + /** * Mapper for {@code _tsid} field included generated when the index is * {@link IndexMode#TIME_SERIES organized into time series}. @@ -208,9 +214,17 @@ public void postParse(DocumentParserContext context) throws IOException { // We need to add the uid or id to nested Lucene documents so that when a document gets deleted, the nested documents are // also deleted. Usually this happens when the nested document is created (in DocumentParserContext#createNestedContext), but // for time-series indices the _id isn't available at that point. - for (LuceneDocument doc : context.nonRootDocuments()) { - assert doc.getField(IdFieldMapper.NAME) == null; - doc.add(new StringField(IdFieldMapper.NAME, uidEncoded, Field.Store.NO)); + if (context.indexSettings().useTimeSeriesSyntheticId()) { + + // For time-series indices with synthetic _id, copy the doc values fields used to synthesize the _id from the + // parent document into its nested documents. + addSyntheticIdFieldsToNestedDocs(context, timeSeriesId, uidEncoded); + + } else { + for (LuceneDocument nestedDoc : context.nonRootDocuments()) { + assert nestedDoc.getField(IdFieldMapper.NAME) == null; + nestedDoc.add(standardIdField(uidEncoded, Field.Store.NO)); + } } } @@ -223,6 +237,70 @@ protected String contentType() { return CONTENT_TYPE; } + private void addSyntheticIdFieldsToNestedDocs(DocumentParserContext context, BytesRef timeSeriesId, BytesRef uidEncoded) { + final var nestedDocFields = new ArrayList(4); + // The synthetic id fields are copied from the root document, so they're correct for any nesting depth + LuceneDocument rootParentDoc = null; + + for (LuceneDocument nestedDoc : context.nonRootDocuments()) { + assert nestedDoc.getField(IdFieldMapper.NAME) == null; + assert nestedDoc.getField(TimeSeriesIdFieldMapper.NAME) == null; + assert nestedDoc.getField(DataStreamTimestampFieldMapper.DEFAULT_PATH) == null; + assert nestedDoc.getField(TimeSeriesRoutingHashFieldMapper.NAME) == null; + + if (rootParentDoc != null) { + assert nestedDocFields.size() == 4 : nestedDocFields.size(); + nestedDoc.addAll(nestedDocFields); + continue; + } + rootParentDoc = nestedDoc.getParent(); + assert rootParentDoc != null; + + // _tsid + var parentTsIdField = rootParentDoc.getField(TimeSeriesIdFieldMapper.NAME); + assert parentTsIdField != null; + + final var parentTimeSeriesId = parentTsIdField.binaryValue(); + assert parentTimeSeriesId.equals(timeSeriesId); + assert parentTimeSeriesId.equals(TsidExtractingIdFieldMapper.extractTimeSeriesIdFromSyntheticId(uidEncoded)); + if (this.useDocValuesSkipper) { + nestedDocFields.add(SortedDocValuesField.indexedField(fieldType().name(), parentTimeSeriesId)); + } else { + nestedDocFields.add(new SortedDocValuesField(fieldType().name(), parentTimeSeriesId)); + } + + // @timestamp + var parentTimestampField = rootParentDoc.getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); + assert parentTimestampField != null; + assert parentTimestampField.numericValue() != null; + + final long parentTimestamp = parentTimestampField.numericValue().longValue(); + assert parentTimestamp == TsidExtractingIdFieldMapper.extractTimestampFromSyntheticId(uidEncoded); + if (this.useDocValuesSkipper) { + nestedDocFields.add(SortedNumericDocValuesField.indexedField(DataStreamTimestampFieldMapper.DEFAULT_PATH, parentTimestamp)); + } else { + nestedDocFields.add(new LongField(DataStreamTimestampFieldMapper.DEFAULT_PATH, parentTimestamp, Field.Store.NO)); + } + + // _ts_routing_hash + var parentRoutingHashField = rootParentDoc.getField(TimeSeriesRoutingHashFieldMapper.NAME); + assert parentRoutingHashField != null; + assert parentRoutingHashField.binaryValue() != null; + + final var parentRoutingHash = parentRoutingHashField.binaryValue(); + assert parentRoutingHash.equals( + Uid.encodeId( + TimeSeriesRoutingHashFieldMapper.encode(TsidExtractingIdFieldMapper.extractRoutingHashFromSyntheticId(uidEncoded)) + ) + ); + nestedDocFields.add(new SortedDocValuesField(TimeSeriesRoutingHashFieldMapper.NAME, parentRoutingHash)); + + // (synthetic) _id + nestedDocFields.add(syntheticIdField(uidEncoded)); + nestedDoc.addAll(nestedDocFields); + } + } + /** * Decode the {@code _tsid} into a human readable map. */ @@ -267,4 +345,5 @@ public static BytesReference buildLegacyTsid(RoutingPathFields routingPathFields return out.bytes(); } } + }