diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index d11be8b800599..e0667e8e2597b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -14,6 +14,8 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; @@ -78,6 +80,7 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; @@ -85,6 +88,9 @@ import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; +import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.seqno.LocalCheckpointTracker; import org.elasticsearch.index.seqno.ReplicationTracker; @@ -461,17 +467,22 @@ protected static LuceneDocument testDocument() { } public static ParsedDocument createParsedDoc(String id, String routing) { - return testParsedDocument(id, routing, testDocumentWithTextField(), new BytesArray("{ \"value\" : \"test\" }"), null, false); + return testParsedDocument(id, routing, testDocumentWithTextField(), new BytesArray("{ \"value\" : \"test\" }"), null, false, false); } public static ParsedDocument createParsedDoc(String id, String routing, boolean recoverySource) { + return createParsedDoc(id, routing, recoverySource, false); + } + + public static ParsedDocument createParsedDoc(String id, String routing, boolean recoverySource, boolean syntheticId) { return testParsedDocument( id, routing, testDocumentWithTextField(), new BytesArray("{ \"value\" : \"test\" }"), null, - recoverySource + recoverySource, + syntheticId ); } @@ -482,7 +493,7 @@ protected ParsedDocument testParsedDocument( BytesReference source, CompressedXContent mappingUpdate ) { - return testParsedDocument(id, routing, document, source, mappingUpdate, false); + return testParsedDocument(id, routing, document, source, mappingUpdate, false, false); } protected static ParsedDocument testParsedDocument( @@ -493,7 +504,37 @@ protected static ParsedDocument testParsedDocument( CompressedXContent mappingUpdate, boolean recoverySource ) { - Field idField = new StringField("_id", Uid.encodeId(id), Field.Store.YES); + return testParsedDocument(id, routing, document, source, mappingUpdate, recoverySource, false); + } + + protected static ParsedDocument testParsedDocument( + String id, + String routing, + LuceneDocument document, + BytesReference source, + CompressedXContent mappingUpdate, + boolean recoverySource, + boolean syntheticId + ) { + var uid = Uid.encodeId(id); + final Field idField; + if (syntheticId) { + idField = IdFieldMapper.syntheticIdField(uid); + var timeSeriesId = TsidExtractingIdFieldMapper.extractTimeSeriesIdFromSyntheticId(uid); + var timestamp = TsidExtractingIdFieldMapper.extractTimestampFromSyntheticId(uid); + int routingHash = TsidExtractingIdFieldMapper.extractRoutingHashFromSyntheticId(uid); + + document.add(SortedDocValuesField.indexedField(TimeSeriesIdFieldMapper.NAME, timeSeriesId)); + document.add(SortedNumericDocValuesField.indexedField("@timestamp", timestamp)); + document.add( + new SortedDocValuesField( + TimeSeriesRoutingHashFieldMapper.NAME, + Uid.encodeId(TimeSeriesRoutingHashFieldMapper.encode(routingHash)) + ) + ); + } else { + idField = new StringField("_id", Uid.encodeId(id), Field.Store.YES); + } Field versionField = new NumericDocValuesField("_version", 0); var seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(seqNoIndexOptions); document.add(idField); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java index 4cd3f62d2d8c4..df752384e4134 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; import org.elasticsearch.index.seqno.RetentionLeases; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.EngineResetLock; @@ -64,6 +65,7 @@ import java.io.IOException; import java.nio.file.Path; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -251,7 +253,7 @@ private EngineConfig engineConfig( indexSettings, BigArrays.NON_RECYCLING_INSTANCE ); - final MapperService mapperService = EngineTestCase.createMapperService(); + final MapperService mapperService = EngineTestCase.createMapperService(indexSettings.getSettings(), "{}"); return new EngineConfig( shardIdValue, threadPool, @@ -262,7 +264,7 @@ private EngineConfig engineConfig( newMergePolicy(), indexWriterConfig.getAnalyzer(), indexWriterConfig.getSimilarity(), - new CodecService(null, BigArrays.NON_RECYCLING_INSTANCE, null), + new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE, null), new Engine.EventListener() { @Override public void onFailedEngine(String reason, Exception e) { @@ -779,12 +781,16 @@ private CheckedBiFunction nestedPa public void testProcessOnceOnPrimary() throws Exception { final Settings.Builder settingsBuilder = indexSettings(IndexVersion.current(), 1, 0).put("index.xpack.ccr.following_index", true); + boolean useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && indexMode == IndexMode.TIME_SERIES && randomBoolean(); switch (indexMode) { case STANDARD: break; case TIME_SERIES: settingsBuilder.put("index.mode", "time_series").put("index.routing_path", "foo"); settingsBuilder.put("index.seq_no.index_options", "points_and_doc_values"); + if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { + settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId); + } break; case LOGSDB: settingsBuilder.put("index.mode", IndexMode.LOGSDB.getName()); @@ -803,8 +809,18 @@ public void testProcessOnceOnPrimary() throws Exception { int numOps = between(10, 100); List operations = new ArrayList<>(numOps); for (int i = 0; i < numOps; i++) { - String docId = Integer.toString(between(1, 100)); - ParsedDocument doc = randomBoolean() ? EngineTestCase.createParsedDoc(docId, null) : nestedDocFunc.apply(docId, randomInt(3)); + String docId = useSyntheticId + ? TsidExtractingIdFieldMapper.createSyntheticId( + new BytesRef(Integer.toString(i)), + Instant.now().toEpochMilli(), + randomNonNegativeInt() + ) + : Integer.toString(i); + ParsedDocument doc = randomBoolean() || useSyntheticId + ? EngineTestCase.createParsedDoc(docId, null, false, useSyntheticId) + // The nested docs uses the default mapper to create the document, and that doesn't include all the necessary + // fields for the synthetic id. Hence we just skip using nested docs if the synthetic id is enabled. + : nestedDocFunc.apply(docId, randomInt(3)); if (randomBoolean()) { operations.add( new Engine.Index(