From 6449aebf825d653f24c8642b0badf24b61b18f07 Mon Sep 17 00:00:00 2001 From: Shreyansh Ray Date: Tue, 15 Apr 2025 17:34:48 +0530 Subject: [PATCH 1/5] Add composite directory factory Signed-off-by: Shreyansh Ray --- .../common/settings/IndexScopedSettings.java | 1 + .../org/opensearch/index/IndexModule.java | 37 +++++++++++++++++ .../org/opensearch/index/IndexService.java | 8 ++-- .../DefaultCompositeDirectoryFactory.java | 40 +++++++++++++++++++ .../opensearch/indices/IndicesService.java | 6 +++ .../main/java/org/opensearch/node/Node.java | 18 +++++++++ .../opensearch/plugins/IndexStorePlugin.java | 37 +++++++++++++++++ 7 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/store/DefaultCompositeDirectoryFactory.java diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index d2c336625a3bb..7c937aa310f16 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -197,6 +197,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING, IndexModule.INDEX_STORE_TYPE_SETTING, + IndexModule.INDEX_COMPOSITE_STORE_TYPE_SETTING, IndexModule.INDEX_STORE_PRE_LOAD_SETTING, IndexModule.INDEX_STORE_HYBRID_NIO_EXTENSIONS, IndexModule.INDEX_RECOVERY_TYPE_SETTING, diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index cb52048e45ab9..e4e0d640a1d5c 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -76,6 +76,7 @@ import org.opensearch.index.shard.IndexingOperationListener; import org.opensearch.index.shard.SearchOperationListener; import org.opensearch.index.similarity.SimilarityService; +import org.opensearch.index.store.DefaultCompositeDirectoryFactory; import org.opensearch.index.store.FsDirectoryFactory; import org.opensearch.index.store.remote.directory.RemoteSnapshotDirectoryFactory; import org.opensearch.index.store.remote.filecache.FileCache; @@ -133,6 +134,8 @@ public final class IndexModule { public static final Setting NODE_STORE_ALLOW_MMAP = Setting.boolSetting("node.store.allow_mmap", true, Property.NodeScope); private static final FsDirectoryFactory DEFAULT_DIRECTORY_FACTORY = new FsDirectoryFactory(); + private static final IndexStorePlugin.CompositeDirectoryFactory DEFAULT_COMPOSITE_DIRECTORY_FACTORY = + new DefaultCompositeDirectoryFactory(); private static final IndexStorePlugin.RecoveryStateFactory DEFAULT_RECOVERY_STATE_FACTORY = RecoveryState::new; @@ -144,6 +147,14 @@ public final class IndexModule { Property.NodeScope ); + public static final Setting INDEX_COMPOSITE_STORE_TYPE_SETTING = new Setting<>( + "index.composite_store.type", + "default", + Function.identity(), + Property.IndexScope, + Property.NodeScope + ); + /** * Index setting which used to determine how the data is cached locally fully or partially. */ @@ -240,6 +251,7 @@ public final class IndexModule { private final Set indexEventListeners = new HashSet<>(); private final Map> similarities = new HashMap<>(); private final Map directoryFactories; + private final Map compositeDirectoryFactories; private final SetOnce> forceQueryCacheProvider = new SetOnce<>(); private final List searchOperationListeners = new ArrayList<>(); private final List indexOperationListeners = new ArrayList<>(); @@ -265,6 +277,7 @@ public IndexModule( final EngineFactory engineFactory, final EngineConfigFactory engineConfigFactory, final Map directoryFactories, + final Map compositeDirectoryFactories, final BooleanSupplier allowExpensiveQueries, final IndexNameExpressionResolver expressionResolver, final Map recoveryStateFactories, @@ -278,6 +291,7 @@ public IndexModule( this.searchOperationListeners.add(new SearchSlowLog(indexSettings)); this.indexOperationListeners.add(new IndexingSlowLog(indexSettings)); this.directoryFactories = Collections.unmodifiableMap(directoryFactories); + this.compositeDirectoryFactories = Collections.unmodifiableMap(compositeDirectoryFactories); this.allowExpensiveQueries = allowExpensiveQueries; this.expressionResolver = expressionResolver; this.recoveryStateFactories = recoveryStateFactories; @@ -301,6 +315,7 @@ public IndexModule( engineFactory, engineConfigFactory, directoryFactories, + null, allowExpensiveQueries, expressionResolver, recoveryStateFactories, @@ -699,6 +714,10 @@ public IndexService newIndexService( .get() == null ? (shard) -> null : indexReaderWrapper.get(); eventListener.beforeIndexCreated(indexSettings.getIndex(), indexSettings.getSettings()); final IndexStorePlugin.DirectoryFactory directoryFactory = getDirectoryFactory(indexSettings, directoryFactories); + final IndexStorePlugin.CompositeDirectoryFactory compositeDirectoryFactory = getCompositeDirectoryFactory( + indexSettings, + compositeDirectoryFactories + ); final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory = getRecoveryStateFactory(indexSettings, recoveryStateFactories); QueryCache queryCache = null; IndexAnalyzers indexAnalyzers = null; @@ -735,6 +754,7 @@ public IndexService newIndexService( client, queryCache, directoryFactory, + compositeDirectoryFactory, remoteDirectoryFactory, eventListener, readerWrapperFactory, @@ -800,6 +820,23 @@ private static IndexStorePlugin.DirectoryFactory getDirectoryFactory( return factory; } + private static IndexStorePlugin.CompositeDirectoryFactory getCompositeDirectoryFactory( + final IndexSettings indexSettings, + final Map indexStoreFactories + ) { + final String compositeStoreType = indexSettings.getValue(INDEX_COMPOSITE_STORE_TYPE_SETTING); + final IndexStorePlugin.CompositeDirectoryFactory factory; + if (compositeStoreType.isEmpty()) { + factory = DEFAULT_COMPOSITE_DIRECTORY_FACTORY; + } else { + factory = indexStoreFactories.get(compositeStoreType); + if (factory == null) { + throw new IllegalArgumentException("Unknown composite store type [" + compositeStoreType + "]"); + } + } + return factory; + } + private static IndexStorePlugin.RecoveryStateFactory getRecoveryStateFactory( final IndexSettings indexSettings, final Map recoveryStateFactories diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 74f579b7fb2d0..8c6a53f9d298f 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -93,7 +93,6 @@ import org.opensearch.index.shard.ShardNotInPrimaryModeException; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.similarity.SimilarityService; -import org.opensearch.index.store.CompositeDirectory; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.Store; import org.opensearch.index.store.remote.filecache.FileCache; @@ -153,6 +152,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final NodeEnvironment nodeEnv; private final ShardStoreDeleter shardStoreDeleter; private final IndexStorePlugin.DirectoryFactory directoryFactory; + private final IndexStorePlugin.CompositeDirectoryFactory compositeDirectoryFactory; private final IndexStorePlugin.DirectoryFactory remoteDirectoryFactory; private final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory; private final CheckedFunction readerWrapper; @@ -221,6 +221,7 @@ public IndexService( Client client, QueryCache queryCache, IndexStorePlugin.DirectoryFactory directoryFactory, + IndexStorePlugin.CompositeDirectoryFactory compositeDirectoryFactory, IndexStorePlugin.DirectoryFactory remoteDirectoryFactory, IndexEventListener eventListener, Function> wrapperFactory, @@ -305,6 +306,7 @@ public IndexService( this.eventListener = eventListener; this.nodeEnv = nodeEnv; this.directoryFactory = directoryFactory; + this.compositeDirectoryFactory = compositeDirectoryFactory; this.remoteDirectoryFactory = remoteDirectoryFactory; this.recoveryStateFactory = recoveryStateFactory; this.engineFactory = Objects.requireNonNull(engineFactory); @@ -399,6 +401,7 @@ public IndexService( client, queryCache, directoryFactory, + null, remoteDirectoryFactory, eventListener, wrapperFactory, @@ -676,8 +679,7 @@ protected void closeInternal() { if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_SETTING) && // TODO : Need to remove this check after support for hot indices is added in Composite Directory this.indexSettings.isWarmIndex()) { - Directory localDirectory = directoryFactory.newDirectory(this.indexSettings, path); - directory = new CompositeDirectory(localDirectory, remoteDirectory, fileCache); + directory = compositeDirectoryFactory.newDirectory(this.indexSettings, path, directoryFactory, remoteDirectory, fileCache); } else { directory = directoryFactory.newDirectory(this.indexSettings, path); } diff --git a/server/src/main/java/org/opensearch/index/store/DefaultCompositeDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/DefaultCompositeDirectoryFactory.java new file mode 100644 index 0000000000000..b7b7e15f6c35b --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/DefaultCompositeDirectoryFactory.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.Directory; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.remote.filecache.FileCache; +import org.opensearch.plugins.IndexStorePlugin; + +import java.io.IOException; + +/** + * Default composite directory factory + */ +public class DefaultCompositeDirectoryFactory implements IndexStorePlugin.CompositeDirectoryFactory { + + private static final Logger logger = LogManager.getLogger(DefaultCompositeDirectoryFactory.class); + + @Override + public Directory newDirectory( + IndexSettings indexSettings, + ShardPath shardPath, + IndexStorePlugin.DirectoryFactory localDirectoryFactory, + Directory remoteDirectory, + FileCache fileCache + ) throws IOException { + logger.trace("Creating composite directory from core - Default CompositeDirectoryFactory"); + Directory localDirectory = localDirectoryFactory.newDirectory(indexSettings, shardPath); + return new CompositeDirectory(localDirectory, remoteDirectory, fileCache); + } +} diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 3c6a4f9e30e9d..508231d1b549e 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -387,6 +387,7 @@ public class IndicesService extends AbstractLifecycleComponent private final MetaStateService metaStateService; private final Collection>> engineFactoryProviders; private final Map directoryFactories; + private final Map compositeDirectoryFactories; private final Map ingestionConsumerFactories; private final Map recoveryStateFactories; final AbstractRefCounted indicesRefCount; // pkg-private for testing @@ -438,6 +439,7 @@ public IndicesService( MetaStateService metaStateService, Collection>> engineFactoryProviders, Map directoryFactories, + Map compositeDirectoryFactories, ValuesSourceRegistry valuesSourceRegistry, Map recoveryStateFactories, IndexStorePlugin.DirectoryFactory remoteDirectoryFactory, @@ -502,6 +504,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.engineFactoryProviders = engineFactoryProviders; this.directoryFactories = directoryFactories; + this.compositeDirectoryFactories = compositeDirectoryFactories; this.recoveryStateFactories = recoveryStateFactories; this.ingestionConsumerFactories = ingestionConsumerFactories; // doClose() is called when shutting down a node, yet there might still be ongoing requests @@ -633,6 +636,7 @@ public IndicesService( metaStateService, engineFactoryProviders, directoryFactories, + null, valuesSourceRegistry, recoveryStateFactories, remoteDirectoryFactory, @@ -1048,6 +1052,7 @@ private synchronized IndexService createIndexService( getEngineFactory(idxSettings), getEngineConfigFactory(idxSettings), directoryFactories, + compositeDirectoryFactories, () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories, @@ -1166,6 +1171,7 @@ public synchronized MapperService createIndexMapperService(IndexMetadata indexMe getEngineFactory(idxSettings), getEngineConfigFactory(idxSettings), directoryFactories, + compositeDirectoryFactories, () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories, diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 4137f1b37de2a..0d007167f6b97 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -159,6 +159,7 @@ import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.remote.RemoteIndexPathUploader; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; +import org.opensearch.index.store.DefaultCompositeDirectoryFactory; import org.opensearch.index.store.IndexStoreListener; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.remote.filecache.FileCache; @@ -894,6 +895,22 @@ protected Node( }); directoryFactories.putAll(builtInDirectoryFactories); + final Map compositeDirectoryFactories = new HashMap<>(); + pluginsService.filterPlugins(IndexStorePlugin.class) + .stream() + .map(IndexStorePlugin::getCompositeDirectoryFactories) + .flatMap(m -> m.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + .forEach((k, v) -> { + if (k.equals("default")) { + throw new IllegalStateException( + "registered composite index store type [" + k + "] conflicts with a built-in default type" + ); + } + compositeDirectoryFactories.put(k, v); + }); + compositeDirectoryFactories.put("default", new DefaultCompositeDirectoryFactory()); + final Map recoveryStateFactories = pluginsService.filterPlugins( IndexStorePlugin.class ) @@ -952,6 +969,7 @@ protected Node( metaStateService, engineFactoryProviders, Map.copyOf(directoryFactories), + Map.copyOf(compositeDirectoryFactories), searchModule.getValuesSourceRegistry(), recoveryStateFactories, remoteDirectoryFactory, diff --git a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java index f0df8a122ed7d..83cbca16295ea 100644 --- a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java @@ -36,10 +36,12 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.Nullable; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.annotation.PublicApi; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.IndexStoreListener; +import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.indices.recovery.RecoveryState; import java.io.IOException; @@ -82,6 +84,41 @@ interface DirectoryFactory { */ Map getDirectoryFactories(); + /** + * An interface that describes how to create a new composite directory instance per shard. + * + * @opensearch.api + */ + @FunctionalInterface + @ExperimentalApi + interface CompositeDirectoryFactory { + /** + * Creates a new composite directory per shard + * @param indexSettings the shards index settings + * @param shardPath the path the shard is using + * @return a new composite directory instance + * @throws IOException if an IOException occurs while opening the directory + */ + Directory newDirectory( + IndexSettings indexSettings, + ShardPath shardPath, + DirectoryFactory localDirectoryFactory, + Directory remoteDirectory, + FileCache fileCache + ) throws IOException; + } + + /** + * The {@link CompositeDirectoryFactory} mappings for this plugin. When an index is created the composite store type setting + * {@link org.opensearch.index.IndexModule#INDEX_COMPOSITE_STORE_TYPE_SETTING} on the index will be examined and either use the default or a + * built-in type, or looked up among all the composite directory factories from {@link IndexStorePlugin} plugins. + * + * @return a map from composite store type to a composite directory factory + */ + default Map getCompositeDirectoryFactories() { + return Collections.emptyMap(); + } + /** * An interface that allows to create a new {@link RecoveryState} per shard. * From ef7f0dcf5119805f96f0b19a01732f441505c30a Mon Sep 17 00:00:00 2001 From: Shreyansh Ray Date: Fri, 18 Apr 2025 14:46:40 +0530 Subject: [PATCH 2/5] Pass empty map instead of null to avoid test failures Signed-off-by: Shreyansh Ray --- .../main/java/org/opensearch/index/IndexModule.java | 4 ++-- .../opensearch/index/store/CompositeDirectory.java | 12 ++++++------ .../java/org/opensearch/indices/IndicesService.java | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index e4e0d640a1d5c..30bbfbcaf4495 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -315,7 +315,7 @@ public IndexModule( engineFactory, engineConfigFactory, directoryFactories, - null, + Collections.emptyMap(), allowExpensiveQueries, expressionResolver, recoveryStateFactories, @@ -826,7 +826,7 @@ private static IndexStorePlugin.CompositeDirectoryFactory getCompositeDirectoryF ) { final String compositeStoreType = indexSettings.getValue(INDEX_COMPOSITE_STORE_TYPE_SETTING); final IndexStorePlugin.CompositeDirectoryFactory factory; - if (compositeStoreType.isEmpty()) { + if (compositeStoreType.isEmpty() || indexStoreFactories.isEmpty()) { factory = DEFAULT_COMPOSITE_DIRECTORY_FACTORY; } else { factory = indexStoreFactories.get(compositeStoreType); diff --git a/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java b/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java index 4733fbcdb0494..aabe553b4e3eb 100644 --- a/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java @@ -52,10 +52,10 @@ @ExperimentalApi public class CompositeDirectory extends FilterDirectory { private static final Logger logger = LogManager.getLogger(CompositeDirectory.class); - private final FSDirectory localDirectory; - private final RemoteSegmentStoreDirectory remoteDirectory; - private final FileCache fileCache; - private final TransferManager transferManager; + protected final FSDirectory localDirectory; + protected final RemoteSegmentStoreDirectory remoteDirectory; + protected final FileCache fileCache; + protected final TransferManager transferManager; /** * Constructor to initialise the composite directory @@ -96,7 +96,7 @@ private String[] listLocalFiles() throws IOException { * @return A list of file names, including the original file (if present) and all its block files. * @throws IOException in case of I/O error while listing files. */ - private List listBlockFiles(String fileName) throws IOException { + protected List listBlockFiles(String fileName) throws IOException { return Stream.of(listLocalFiles()) .filter(file -> file.equals(fileName) || file.startsWith(fileName + FileTypeUtils.BLOCK_FILE_IDENTIFIER)) .collect(Collectors.toList()); @@ -383,7 +383,7 @@ private String[] getRemoteFiles() throws IOException { return remoteFiles; } - private void cacheFile(String name) throws IOException { + protected void cacheFile(String name) throws IOException { Path filePath = getFilePath(name); // put will increase the refCount for the path, making sure it is not evicted, will decrease the ref after it is uploaded to Remote // so that it can be evicted after that diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 508231d1b549e..8b2c0547ca35e 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -636,7 +636,7 @@ public IndicesService( metaStateService, engineFactoryProviders, directoryFactories, - null, + Collections.emptyMap(), valuesSourceRegistry, recoveryStateFactories, remoteDirectoryFactory, From c7805e1c9c50af91672f41e584461a780107c632 Mon Sep 17 00:00:00 2001 From: Shreyansh Ray Date: Tue, 22 Apr 2025 15:25:35 +0530 Subject: [PATCH 3/5] Add tests for DefaultCompositeDirectoryFactory Signed-off-by: Shreyansh Ray --- ...DefaultCompositeDirectoryFactoryTests.java | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 server/src/test/java/org/opensearch/index/store/DefaultCompositeDirectoryFactoryTests.java diff --git a/server/src/test/java/org/opensearch/index/store/DefaultCompositeDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/DefaultCompositeDirectoryFactoryTests.java new file mode 100644 index 0000000000000..7be4a30516ed6 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/DefaultCompositeDirectoryFactoryTests.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.remote.filecache.FileCache; +import org.opensearch.index.store.remote.filecache.FileCacheFactory; +import org.opensearch.plugins.IndexStorePlugin; +import org.opensearch.test.IndexSettingsModule; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DefaultCompositeDirectoryFactoryTests extends BaseRemoteSegmentStoreDirectoryTests { + + private DefaultCompositeDirectoryFactory directoryFactory; + private IndexSettings indexSettings; + private ShardPath shardPath; + private IndexStorePlugin.DirectoryFactory localDirectoryFactory; + private FSDirectory localDirectory; + private FileCache fileCache; + + @Before + public void setup() throws IOException { + indexSettings = IndexSettingsModule.newIndexSettings("foo", Settings.builder().build()); + Path tempDir = createTempDir().resolve(indexSettings.getUUID()).resolve("0"); + shardPath = new ShardPath(false, tempDir, tempDir, new ShardId(indexSettings.getIndex(), 0)); + localDirectoryFactory = mock(IndexStorePlugin.DirectoryFactory.class); + localDirectory = FSDirectory.open(createTempDir()); + fileCache = FileCacheFactory.createConcurrentLRUFileCache(10000, new NoopCircuitBreaker(CircuitBreaker.REQUEST)); + when(localDirectoryFactory.newDirectory(indexSettings, shardPath)).thenReturn(localDirectory); + setupRemoteSegmentStoreDirectory(); + populateMetadata(); + remoteSegmentStoreDirectory.init(); + } + + public void testNewDirectory() throws IOException { + directoryFactory = new DefaultCompositeDirectoryFactory(); + Directory directory = directoryFactory.newDirectory( + indexSettings, + shardPath, + localDirectoryFactory, + remoteSegmentStoreDirectory, + fileCache + ); + assertNotNull(directory); + assert (directory instanceof CompositeDirectory); + verify(localDirectoryFactory).newDirectory(indexSettings, shardPath); + } + +} From 03aecdc5cb0d0490230d14ef64dc5573d33580f8 Mon Sep 17 00:00:00 2001 From: Shreyansh Ray Date: Thu, 24 Apr 2025 18:28:40 +0530 Subject: [PATCH 4/5] Add changlog and address review comments Signed-off-by: Shreyansh Ray --- CHANGELOG.md | 1 + server/src/main/java/org/opensearch/index/IndexModule.java | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afe3cd983b86c..40db33bf6da16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump OpenSearch Core main branch to 3.0.0 ([#18039](https://github.com/opensearch-project/OpenSearch/pull/18039)) - Update API of Message in index to add the timestamp for lag calculation in ingestion polling ([#17977](https://github.com/opensearch-project/OpenSearch/pull/17977/)) - Enabled default throttling for all tasks submitted to cluster manager ([#17711](https://github.com/opensearch-project/OpenSearch/pull/17711)) +- Add composite directory factory ([#17988](https://github.com/opensearch-project/OpenSearch/pull/17988)) ### Changed - Change the default max header size from 8KB to 16KB. ([#18024](https://github.com/opensearch-project/OpenSearch/pull/18024)) diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 30bbfbcaf4495..9d2f930cabc4a 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -822,14 +822,14 @@ private static IndexStorePlugin.DirectoryFactory getDirectoryFactory( private static IndexStorePlugin.CompositeDirectoryFactory getCompositeDirectoryFactory( final IndexSettings indexSettings, - final Map indexStoreFactories + final Map compositeDirectoryFactories ) { final String compositeStoreType = indexSettings.getValue(INDEX_COMPOSITE_STORE_TYPE_SETTING); final IndexStorePlugin.CompositeDirectoryFactory factory; - if (compositeStoreType.isEmpty() || indexStoreFactories.isEmpty()) { + if (compositeStoreType.isEmpty() || compositeDirectoryFactories.isEmpty()) { factory = DEFAULT_COMPOSITE_DIRECTORY_FACTORY; } else { - factory = indexStoreFactories.get(compositeStoreType); + factory = compositeDirectoryFactories.get(compositeStoreType); if (factory == null) { throw new IllegalArgumentException("Unknown composite store type [" + compositeStoreType + "]"); } From 73d7ae378588a97b84a400c46b378b3408623e05 Mon Sep 17 00:00:00 2001 From: Shreyansh Ray Date: Fri, 25 Apr 2025 11:58:50 +0530 Subject: [PATCH 5/5] Empty commit to trigger build. Signed-off-by: Shreyansh Ray