From 4f9ec4c2157bc80854962f46cf2a5c1c33ba825e Mon Sep 17 00:00:00 2001 From: RS146BIJAY Date: Tue, 9 Dec 2025 07:06:15 +0530 Subject: [PATCH] LeafReader should not remove SubReaderWrappers incase IndexWriter encounters a non aborting Exception Signed-off-by: RS146BIJAY --- CHANGELOG.md | 1 + .../org/opensearch/common/lucene/Lucene.java | 27 +- .../index/engine/InternalEngineTests.java | 418 +++++++++++++++++- .../org/opensearch/index/MapperTestUtils.java | 18 +- .../index/engine/EngineTestCase.java | 115 +++++ 5 files changed, 556 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a21e092dd850d..091d54ed30967 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix `cluster.remote..server_name` setting no populating SNI ([#20321](https://github.com/opensearch-project/OpenSearch/pull/20321)) - Fix X-Opaque-Id header propagation (along with other response headers) for streaming Reactor Netty 4 transport ([#20371](https://github.com/opensearch-project/OpenSearch/pull/20371)) - Fix indexing regression and bug fixes for grouping criteria. ([20145](https://github.com/opensearch-project/OpenSearch/pull/20145)) +- LeafReader should not remove SubReaderWrappers incase IndexWriter encounters a non aborting Exception ([#20193](https://github.com/opensearch-project/OpenSearch/pull/20193)) ### Dependencies - Bump `com.google.auth:google-auth-library-oauth2-http` from 1.38.0 to 1.41.0 ([#20183](https://github.com/opensearch-project/OpenSearch/pull/20183)) diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java index 5eb882cf22983..2b0caf5516d83 100644 --- a/server/src/main/java/org/opensearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/opensearch/common/lucene/Lucene.java @@ -84,6 +84,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.common.Nullable; import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.lucene.index.DerivedSourceLeafReader; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.util.iterable.Iterables; import org.opensearch.core.common.Strings; @@ -951,12 +952,36 @@ public LeafReader wrap(LeafReader leaf) { } assert numDocs == popCount(hardLiveDocs) : numDocs + " != " + popCount(hardLiveDocs); - return new LeafReaderWithLiveDocs(segmentReader, hardLiveDocs, numDocs); + if (isDerivedSourceEnabled(leaf)) { + return new LeafReaderWithLiveDocs(leaf, hardLiveDocs, numDocs); + } else { + return new LeafReaderWithLiveDocs(segmentReader, hardLiveDocs, numDocs); + } } private boolean isContextAwareEnabled(SegmentReader reader) { return reader.getSegmentInfo().info.getAttribute(CriteriaBasedCodec.BUCKET_NAME) != null; } + + /** + * A FilterCodecReader can never accept a DerivedSourceLeafReader as a delegate as it is IndexReader. + * DerivedSourceLeafReader can be wrapped up by only FilterLeafReader. + * @param reader the underlying leafReader. + * + * @return whether derived source is enabled or not. + */ + public boolean isDerivedSourceEnabled(LeafReader reader) { + if (reader instanceof SegmentReader) { + return false; + } else if (reader instanceof DerivedSourceLeafReader) { + return true; + } else if (reader instanceof FilterLeafReader) { + FilterLeafReader filterLeafReader = (FilterLeafReader) reader; + return isDerivedSourceEnabled(filterLeafReader.getDelegate()); + } + + return false; + } }); } diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index 7e1a92a925df9..b12b2e077dd58 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -47,6 +47,8 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FilterDirectoryReader; +import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -8412,7 +8414,7 @@ public void testNewChangesSnapshotWithDerivedSource() throws IOException { final int numDocs = randomIntBetween(1, 100); try (Store store = createStore()) { - EngineConfig engineConfig = createEngineConfigWithMapperSupplierForDerivedSource(store); + EngineConfig engineConfig = createEngineConfigWithMapperSupplierForDerivedSource(store, false); InternalEngine engine = null; try { engine = createEngine(engineConfig); @@ -8475,6 +8477,360 @@ public void testNewChangesSnapshotWithDerivedSource() throws IOException { } } + @LockFeatureFlag(CONTEXT_AWARE_MIGRATION_EXPERIMENTAL_FLAG) + public void testNewChangesSnapshotWithDeleteAndUpdateWithDerivedSourceAndContextAwareEnabled() throws IOException { + IOUtils.close(engine, store); + final List operations = new ArrayList<>(); + int numDocs = randomIntBetween(10, 100); + int numDocsToDelete = randomIntBetween(1, numDocs / 2); + Set deletedDocs = new HashSet<>(); + + try (Store store = createStore()) { + EngineConfig engineConfig = createEngineConfigWithMapperSupplierForDerivedSource(store, true); + InternalEngine engine = null; + try { + engine = createEngine(engineConfig); + // First index documents + for (int i = 0; i < numDocs; i++) { + ParsedDocument doc = testParsedDocument( + Integer.toString(i), + null, + testContextSpecificDocument(), + null, // No source, it should be derived + null + ); + Engine.Index index = new Engine.Index( + newUid(doc), + doc, + UNASSIGNED_SEQ_NO, + primaryTerm.get(), + i, + VersionType.EXTERNAL, + Engine.Operation.Origin.PRIMARY, + System.nanoTime(), + -1, + false, + UNASSIGNED_SEQ_NO, + 0 + ); + operations.add(index); + engine.index(index); + } + + // Delete some documents + for (int i = 0; i < numDocsToDelete; i++) { + String idToDelete = Integer.toString(randomInt(numDocs - 1)); + if (!deletedDocs.contains(idToDelete)) { + final Engine.Delete delete = new Engine.Delete( + idToDelete, + newUid(idToDelete), + UNASSIGNED_SEQ_NO, + primaryTerm.get(), + i + numDocs, + VersionType.EXTERNAL, + Engine.Operation.Origin.PRIMARY, + System.nanoTime(), + UNASSIGNED_SEQ_NO, + 0 + ); + operations.add(delete); + engine.delete(delete); + deletedDocs.add(idToDelete); + } + } + + // Update some remaining documents. + int numDocsToUpdate = randomIntBetween(1, numDocs - deletedDocs.size()); + Set updatedDocs = new HashSet<>(); + for (int i = 0; i < numDocsToUpdate; i++) { + String idToUpdate; + do { + idToUpdate = Integer.toString(randomInt(numDocs - 1)); + } while (deletedDocs.contains(idToUpdate) || updatedDocs.contains(idToUpdate)); + + Document document = testDocumentWithGroupingCriteria(); + document.add(new TextField("value", "updated", Field.Store.YES)); + ParsedDocument doc = testParsedDocument(idToUpdate, null, document, null, null); + Engine.Index update = new Engine.Index( + newUid(doc), + doc, + UNASSIGNED_SEQ_NO, + primaryTerm.get(), + numDocs + numDocsToDelete + i, + VersionType.EXTERNAL, + Engine.Operation.Origin.PRIMARY, + System.nanoTime(), + -1, + false, + UNASSIGNED_SEQ_NO, + 0 + ); + operations.add(update); + engine.index(update); + updatedDocs.add(idToUpdate); + } + + engine.refresh("test"); + + // Test snapshot with all operations + try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", 0, operations.size() - 1, true, true)) { + int count = 0; + Translog.Operation operation; + while ((operation = snapshot.next()) != null) { + if (operation instanceof Translog.Index) { + Translog.Index indexOp = (Translog.Index) operation; + String docId = indexOp.id(); + if (updatedDocs.contains(docId)) { + // Verify updated content using get + try ( + Engine.GetResult get = engine.get( + new Engine.Get(true, true, docId, newUid(docId)), + engine::acquireSearcher + ) + ) { + assertTrue("Document " + docId + " should exist", get.exists()); + StoredFields storedFields = get.docIdAndVersion().reader.storedFields(); + org.apache.lucene.document.Document document = storedFields.document(get.docIdAndVersion().docId); + assertEquals( + "Document " + docId + " should have updated value", + "updated", + document.getField("value").stringValue() + ); + } + } + } else if (operation instanceof Translog.Delete) { + String docId = ((Translog.Delete) operation).id(); + assertTrue("Document " + docId + " should be in deleted set", deletedDocs.contains(docId)); + + // Verify document is actually deleted + try ( + Engine.GetResult get = engine.get( + new Engine.Get(true, false, docId, newUid(docId)), + engine::acquireSearcher + ) + ) { + assertFalse("Document " + docId + " should not exist", get.exists()); + } + } + count++; + } + + // Verify we got all operations + assertEquals("Expected number of operations", operations.size(), count); + } + + // Test snapshot with accurate count + try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", 0, operations.size() - 1, true, true)) { + assertEquals("Total number of operations", operations.size(), snapshot.totalOperations()); + } + + // Test snapshot with specific range + int from = randomIntBetween(0, operations.size() / 2); + int to = randomIntBetween(from, operations.size() - 1); + try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", from, to, false, true)) { + int count = 0; + while (snapshot.next() != null) { + count++; + } + assertEquals("Expected number of operations in range", to - from + 1, count); + } + } finally { + IOUtils.close(engine, store); + } + } + } + + @LockFeatureFlag(CONTEXT_AWARE_MIGRATION_EXPERIMENTAL_FLAG) + public void testNewChangesSnapshotWithDeleteAndUpdateWithDerivedSourceAndContextAwareEnabledWithFilterWrapper() throws IOException { + IOUtils.close(engine, store); + final List operations = new ArrayList<>(); + int numDocs = randomIntBetween(10, 100); + int numDocsToDelete = randomIntBetween(1, numDocs / 2); + Set deletedDocs = new HashSet<>(); + + try (Store store = createStore()) { + EngineConfig engineConfig = createEngineConfigWithMapperSupplierForDerivedSource(store, true); + InternalEngine engine = null; + try { + engine = createEngineForWrapper( + engineConfig, + reader -> new FilterDirectoryReader(reader, new FilterDirectoryReader.SubReaderWrapper() { + @Override + public LeafReader wrap(LeafReader reader) { + return new FilterLeafReader(reader) { + @Override + public CacheHelper getCoreCacheHelper() { + return in.getCoreCacheHelper(); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); + } + }; + } + }) { + @Override + protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { + return in; + } + + @Override + public CacheHelper getReaderCacheHelper() { + return reader.getReaderCacheHelper(); + } + } + ); + // First index documents + for (int i = 0; i < numDocs; i++) { + ParsedDocument doc = testParsedDocument( + Integer.toString(i), + null, + testContextSpecificDocument(), + null, // No source, it should be derived + null + ); + Engine.Index index = new Engine.Index( + newUid(doc), + doc, + UNASSIGNED_SEQ_NO, + primaryTerm.get(), + i, + VersionType.EXTERNAL, + Engine.Operation.Origin.PRIMARY, + System.nanoTime(), + -1, + false, + UNASSIGNED_SEQ_NO, + 0 + ); + operations.add(index); + engine.index(index); + } + + // Delete some documents + for (int i = 0; i < numDocsToDelete; i++) { + String idToDelete = Integer.toString(randomInt(numDocs - 1)); + if (!deletedDocs.contains(idToDelete)) { + final Engine.Delete delete = new Engine.Delete( + idToDelete, + newUid(idToDelete), + UNASSIGNED_SEQ_NO, + primaryTerm.get(), + i + numDocs, + VersionType.EXTERNAL, + Engine.Operation.Origin.PRIMARY, + System.nanoTime(), + UNASSIGNED_SEQ_NO, + 0 + ); + operations.add(delete); + engine.delete(delete); + deletedDocs.add(idToDelete); + } + } + + // Update some remaining documents. + int numDocsToUpdate = randomIntBetween(1, numDocs - deletedDocs.size()); + Set updatedDocs = new HashSet<>(); + for (int i = 0; i < numDocsToUpdate; i++) { + String idToUpdate; + do { + idToUpdate = Integer.toString(randomInt(numDocs - 1)); + } while (deletedDocs.contains(idToUpdate) || updatedDocs.contains(idToUpdate)); + + Document document = testDocumentWithGroupingCriteria(); + document.add(new TextField("value", "updated", Field.Store.YES)); + ParsedDocument doc = testParsedDocument(idToUpdate, null, document, null, null); + Engine.Index update = new Engine.Index( + newUid(doc), + doc, + UNASSIGNED_SEQ_NO, + primaryTerm.get(), + numDocs + numDocsToDelete + i, + VersionType.EXTERNAL, + Engine.Operation.Origin.PRIMARY, + System.nanoTime(), + -1, + false, + UNASSIGNED_SEQ_NO, + 0 + ); + operations.add(update); + engine.index(update); + updatedDocs.add(idToUpdate); + } + + engine.refresh("test"); + + // Test snapshot with all operations + try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", 0, operations.size() - 1, true, true)) { + int count = 0; + Translog.Operation operation; + while ((operation = snapshot.next()) != null) { + if (operation instanceof Translog.Index) { + Translog.Index indexOp = (Translog.Index) operation; + String docId = indexOp.id(); + if (updatedDocs.contains(docId)) { + // Verify updated content using get + try ( + Engine.GetResult get = engine.get( + new Engine.Get(true, true, docId, newUid(docId)), + engine::acquireSearcher + ) + ) { + assertTrue("Document " + docId + " should exist", get.exists()); + StoredFields storedFields = get.docIdAndVersion().reader.storedFields(); + org.apache.lucene.document.Document document = storedFields.document(get.docIdAndVersion().docId); + assertEquals( + "Document " + docId + " should have updated value", + "updated", + document.getField("value").stringValue() + ); + } + } + } else if (operation instanceof Translog.Delete) { + String docId = ((Translog.Delete) operation).id(); + assertTrue("Document " + docId + " should be in deleted set", deletedDocs.contains(docId)); + + // Verify document is actually deleted + try ( + Engine.GetResult get = engine.get( + new Engine.Get(true, false, docId, newUid(docId)), + engine::acquireSearcher + ) + ) { + assertFalse("Document " + docId + " should not exist", get.exists()); + } + } + count++; + } + + // Verify we got all operations + assertEquals("Expected number of operations", operations.size(), count); + } + + // Test snapshot with accurate count + try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", 0, operations.size() - 1, true, true)) { + assertEquals("Total number of operations", operations.size(), snapshot.totalOperations()); + } + + // Test snapshot with specific range + int from = randomIntBetween(0, operations.size() / 2); + int to = randomIntBetween(from, operations.size() - 1); + try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", from, to, false, true)) { + int count = 0; + while (snapshot.next() != null) { + count++; + } + assertEquals("Expected number of operations in range", to - from + 1, count); + } + } finally { + IOUtils.close(engine, store); + } + } + } + public void testNewChangesSnapshotWithDeleteAndUpdateWithDerivedSource() throws IOException { IOUtils.close(engine, store); final List operations = new ArrayList<>(); @@ -8483,7 +8839,7 @@ public void testNewChangesSnapshotWithDeleteAndUpdateWithDerivedSource() throws Set deletedDocs = new HashSet<>(); try (Store store = createStore()) { - EngineConfig engineConfig = createEngineConfigWithMapperSupplierForDerivedSource(store); + EngineConfig engineConfig = createEngineConfigWithMapperSupplierForDerivedSource(store, false); InternalEngine engine = null; try { engine = createEngine(engineConfig); @@ -8778,30 +9134,54 @@ public void testShardFailsForCompositeIndexWriterInCaseAddIndexesThrewExceptionW } } - private EngineConfig createEngineConfigWithMapperSupplierForDerivedSource(Store store) throws IOException { + private EngineConfig createEngineConfigWithMapperSupplierForDerivedSource(Store store, boolean contextAwareEnabled) throws IOException { // Setup with derived source enabled - Settings settings = Settings.builder() + XContentBuilder mapping; + Settings.Builder settingBuilder = Settings.builder() .put(defaultSettings.getSettings()) .put(IndexSettings.INDEX_DERIVED_SOURCE_SETTING.getKey(), true) - .put("index.refresh_interval", -1) - .build(); + .put("index.refresh_interval", -1); + + if (contextAwareEnabled == true) { + settingBuilder.put(IndexSettings.INDEX_CONTEXT_AWARE_ENABLED_SETTING.getKey(), true); + mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("value") + .field("type", "text") + .field("store", true) + .endObject() + .endObject() + .startObject("context_aware_grouping") + .array("fields", "value") + .startObject("script") + .field("source", "String.valueOf(grouping_criteria)") + .endObject() + .endObject() + .endObject() + .endObject(); + } else { + mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("value") + .field("type", "text") + .field("store", true) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + Settings settings = settingBuilder.build(); IndexMetadata indexMetadata = IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build(); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetadata); // Create mapping with required fields - XContentBuilder mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject("properties") - .startObject("value") - .field("type", "text") - .field("store", true) - .endObject() - .endObject() - .endObject() - .endObject(); - - final MapperService mapperService = createMapperService(); + + final MapperService mapperService = createMapperServiceForContextAwareIndex(); mapperService.merge("_doc", new CompressedXContent(mapping.toString()), MapperService.MergeReason.MAPPING_UPDATE); final DocumentMapper documentMapper = mapperService.documentMapper(); DocumentMapperForType documentMapperForType = new DocumentMapperForType(documentMapper, null); diff --git a/test/framework/src/main/java/org/opensearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/opensearch/index/MapperTestUtils.java index 302180fcf95df..32183630386a6 100644 --- a/test/framework/src/main/java/org/opensearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/opensearch/index/MapperTestUtils.java @@ -48,6 +48,7 @@ import org.opensearch.indices.IndicesModule; import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.plugins.AnalysisPlugin; +import org.opensearch.script.ScriptService; import org.opensearch.test.IndexSettingsModule; import java.io.IOException; @@ -66,9 +67,19 @@ public static MapperService newMapperService( Path tempDir, Settings indexSettings, String indexName + ) throws IOException { + return newMapperService(xContentRegistry, tempDir, indexSettings, indexName, null); + } + + public static MapperService newMapperService( + NamedXContentRegistry xContentRegistry, + Path tempDir, + Settings indexSettings, + String indexName, + ScriptService scriptService ) throws IOException { IndicesModule indicesModule = new IndicesModule(Collections.emptyList()); - return newMapperService(xContentRegistry, tempDir, indexSettings, indicesModule, indexName); + return newMapperService(xContentRegistry, tempDir, indexSettings, indicesModule, indexName, scriptService); } public static MapperService newMapperService( @@ -76,7 +87,8 @@ public static MapperService newMapperService( Path tempDir, Settings settings, IndicesModule indicesModule, - String indexName + String indexName, + ScriptService scriptService ) throws IOException { Settings.Builder settingsBuilder = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), tempDir).put(settings); if (settings.get(IndexMetadata.SETTING_VERSION_CREATED) == null) { @@ -95,7 +107,7 @@ public static MapperService newMapperService( mapperRegistry, () -> null, () -> false, - null + scriptService ); } diff --git a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java index e4e56cddd21d2..2220be6daba81 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java @@ -62,6 +62,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.replication.ReplicationResponse; @@ -70,6 +71,7 @@ import org.opensearch.cluster.routing.AllocationId; import org.opensearch.cluster.service.ClusterApplierService; import org.opensearch.common.CheckedBiFunction; +import org.opensearch.common.CheckedFunction; import org.opensearch.common.Nullable; import org.opensearch.common.Randomness; import org.opensearch.common.compress.CompressedXContent; @@ -112,6 +114,7 @@ import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.FsDirectoryFactory; import org.opensearch.index.store.Store; @@ -123,6 +126,13 @@ import org.opensearch.index.translog.TranslogManager; import org.opensearch.index.translog.TranslogOperationHelper; import org.opensearch.index.translog.listener.TranslogEventListener; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.ScriptPlugin; +import org.opensearch.script.MockScriptEngine; +import org.opensearch.script.ScriptContext; +import org.opensearch.script.ScriptEngine; +import org.opensearch.script.ScriptModule; +import org.opensearch.script.ScriptService; import org.opensearch.test.DummyShardLock; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; @@ -136,8 +146,10 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -226,6 +238,7 @@ public void setUp() throws Exception { } else { codecName = "default"; } + defaultSettings = IndexSettingsModule.newIndexSettings("test", indexSettings()); threadPool = new TestThreadPool(getClass().getName()); store = createStore(); @@ -382,6 +395,12 @@ protected static ParseContext.Document testDocumentWithTextField(String value) { return document; } + protected static ParseContext.Document testDocumentWithGroupingCriteria() { + ParseContext.Document document = new ParseContext.Document(); + document.setGroupingCriteria("grouping_criteria"); + return document; + } + protected static ParseContext.Document testDocument() { return new ParseContext.Document(); } @@ -695,6 +714,43 @@ protected InternalEngine createEngine(EngineConfig config) throws IOException { return createEngine(null, null, null, config); } + protected InternalEngine createEngineForWrapper( + EngineConfig config, + CheckedFunction readerWrapper + ) throws IOException { + final Store store = config.getStore(); + final Directory directory = store.directory(); + if (Lucene.indexExists(directory) == false) { + store.createEmpty(config.getIndexSettings().getIndexVersionCreated().luceneVersion); + final String translogUuid = Translog.createEmptyTranslog( + config.getTranslogConfig().getTranslogPath(), + SequenceNumbers.NO_OPS_PERFORMED, + shardId, + primaryTerm.get() + ); + store.associateIndexWithNewTranslog(translogUuid); + + } + + InternalEngine internalEngine = new InternalEngine(config) { + @Override + public Searcher acquireSearcher(String source, SearcherScope scope, Function wrapper) + throws EngineException { + try { + Searcher searcher = super.acquireSearcher(source, scope, wrapper); + return IndexShard.wrapSearcher(searcher, readerWrapper); + } catch (IOException ex) { + throw new OpenSearchException("failed to wrap searcher", ex); + } + + } + }; + + translogHandler = createTranslogHandler(config.getIndexSettings(), internalEngine); + internalEngine.translogManager().recoverFromTranslog(translogHandler, internalEngine.getProcessedLocalCheckpoint(), Long.MAX_VALUE); + return internalEngine; + } + protected InternalEngine createEngine( @Nullable IndexWriterFactory indexWriterFactory, @Nullable BiFunction localCheckpointTrackerSupplier, @@ -1606,10 +1662,69 @@ public static void assertAtMostOneLuceneDocumentPerSequenceNumber(IndexSettings } } + public static MapperService createMapperServiceForContextAwareIndex() throws IOException { + String mapping = "{\"properties\": {}}"; + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.INDEX_CONTEXT_AWARE_ENABLED_SETTING.getKey(), true) + ) + .putMapping(mapping) + .build(); + + ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, Collections.singletonList(new ContextAwareCustomScriptPlugin())); + ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts); + + MapperService mapperService = MapperTestUtils.newMapperService( + new NamedXContentRegistry(ClusterModule.getNamedXWriteables()), + createTempDir(), + Settings.EMPTY, + "test", + scriptService + ); + mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_UPDATE); + return mapperService; + + } + public static MapperService createMapperService() throws IOException { return createMapperService("{\"properties\": {}}"); } + public static class ContextAwareCustomScriptPlugin extends Plugin implements ScriptPlugin { + + @SuppressWarnings("unchecked") + protected Map, Object>> pluginScripts() { + Map, Object>> pluginScripts = new HashMap<>(); + pluginScripts.put("ctx.op='delete'", vars -> ((Map) vars.get("ctx")).put("op", "delete")); + pluginScripts.put("String.valueOf(grouping_criteria)", vars -> "grouping_criteria"); + + return pluginScripts; + } + + public static final String NAME = "painless"; + + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new MockScriptEngine(pluginScriptLang(), pluginScripts(), nonDeterministicPluginScripts(), pluginContextCompilers()); + } + + protected Map, Object>> nonDeterministicPluginScripts() { + return Collections.emptyMap(); + } + + protected Map, MockScriptEngine.ContextCompiler> pluginContextCompilers() { + return Collections.emptyMap(); + } + + public String pluginScriptLang() { + return NAME; + } + } + public static MapperService createMapperService(String mapping) throws IOException { IndexMetadata indexMetadata = IndexMetadata.builder("test") .settings(