Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.opensearch.index.engine.exec.EngineReaderManager;
import org.opensearch.index.engine.exec.commit.Committer;
import org.opensearch.index.engine.exec.commit.CommitterFactory;
import org.opensearch.index.store.FormatChecksumStrategy;
import org.opensearch.plugins.EnginePlugin;
import org.opensearch.plugins.Plugin;
import org.opensearch.plugins.SearchBackEndPlugin;
Expand Down Expand Up @@ -66,15 +65,11 @@ public DataFormat getDataFormat() {
* Requires the committer to be a {@link LuceneCommitter}.
*
* @param indexingEngineConfig the engine configuration containing committer, mapper service, and store
* @param checksumStrategy the checksum strategy for the format (unused by Lucene)
* @return a new Lucene indexing execution engine
* @throws IllegalStateException if the committer is not a {@link LuceneCommitter}
*/
@Override
public IndexingExecutionEngine<?, ?> indexingEngine(
IndexingEngineConfig indexingEngineConfig,
FormatChecksumStrategy checksumStrategy
) {
public IndexingExecutionEngine<?, ?> indexingEngine(IndexingEngineConfig indexingEngineConfig) {
Committer committer = indexingEngineConfig.committer();
if (committer instanceof LuceneCommitter luceneCommitter) {
return new LuceneIndexingExecutionEngine(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ private CommitterConfig createCommitterConfig() throws IOException {
null,
null,
null,
null,
null
);
return new CommitterConfig(engineConfig);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ private LuceneCommitter createCommitter() throws IOException {
null,
null,
null,
null,
null
);
CommitterConfig settings = new CommitterConfig(engineConfig);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.opensearch.index.engine.dataformat.DataFormatRegistry;
import org.opensearch.index.engine.dataformat.IndexingEngineConfig;
import org.opensearch.index.engine.dataformat.IndexingExecutionEngine;
import org.opensearch.index.store.FormatChecksumStrategy;
import org.opensearch.plugins.ExtensiblePlugin;
import org.opensearch.plugins.Plugin;

Expand Down Expand Up @@ -92,19 +91,14 @@ public DataFormat getDataFormat() {
}

@Override
public IndexingExecutionEngine<?, ?> indexingEngine(IndexingEngineConfig settings, FormatChecksumStrategy checksumStrategy) {
Map<String, FormatChecksumStrategy> strategies = new HashMap<>();
for (Map.Entry<String, DataFormatDescriptor> entry : getFormatDescriptors(settings.indexSettings(), settings.registry())
.entrySet()) {
strategies.put(entry.getKey(), entry.getValue().getChecksumStrategy());
}
public IndexingExecutionEngine<?, ?> indexingEngine(IndexingEngineConfig settings) {
return new CompositeIndexingExecutionEngine(
settings.indexSettings(),
settings.mapperService(),
settings.committer(),
settings.registry(),
settings.store(),
strategies
settings.checksumStrategies()
);
}

Expand All @@ -116,11 +110,11 @@ public Map<String, DataFormatDescriptor> getFormatDescriptors(IndexSettings inde

Map<String, DataFormatDescriptor> descriptors = new HashMap<>();
if (primaryFormatName != null) {
descriptors.putAll(dataFormatRegistry.getFormatDescriptors(indexSettings));
descriptors.putAll(dataFormatRegistry.getFormatDescriptors(indexSettings, dataFormatRegistry.format(primaryFormatName)));
}
for (String secondaryName : secondaryFormatNames) {
if (secondaryName != null) {
descriptors.putAll(dataFormatRegistry.getFormatDescriptors(indexSettings));
descriptors.putAll(dataFormatRegistry.getFormatDescriptors(indexSettings, dataFormatRegistry.format(secondaryName)));
}
}
return Map.copyOf(descriptors);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,14 @@ public CompositeIndexingExecutionEngine(
validateFormatsRegistered(dataFormatRegistry, primaryFormatName, secondaryFormatNames);

Map<String, FormatChecksumStrategy> strategies = checksumStrategies != null ? checksumStrategies : Map.of();
IndexingEngineConfig engineSettings = new IndexingEngineConfig(committer, mapperService, indexSettings, store, dataFormatRegistry);
IndexingEngineConfig engineSettings = new IndexingEngineConfig(
committer,
mapperService,
indexSettings,
store,
dataFormatRegistry,
strategies
);

List<DataFormat> allFormats = new ArrayList<>();
DataFormat primaryFormat = dataFormatRegistry.format(primaryFormatName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.engine.dataformat.DataFormat;
import org.opensearch.index.engine.dataformat.DataFormatRegistry;
import org.opensearch.test.OpenSearchTestCase;

Expand Down Expand Up @@ -60,8 +61,10 @@ public void testGetFormatDescriptorsDelegatestoPlugins() {
IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY);

DataFormatRegistry registry = mock(DataFormatRegistry.class);
when(registry.format("parquet")).thenReturn(CompositeTestHelper.stubFormat("parquet", 2, java.util.Set.of()));
when(registry.getFormatDescriptors(indexSettings)).thenReturn(
DataFormat parquetFormat = CompositeTestHelper.stubFormat("parquet", 2, java.util.Set.of());
when(registry.format("parquet")).thenReturn(parquetFormat);
when(registry.format("lucene")).thenReturn(CompositeTestHelper.stubFormat("lucene", 1, java.util.Set.of()));
when(registry.getFormatDescriptors(indexSettings, parquetFormat)).thenReturn(
Map.of(
"parquet",
new org.opensearch.index.engine.dataformat.DataFormatDescriptor(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public void testConstructorThrowsWhenSecondaryFormatNotRegistered() {
when(registry.getRegisteredFormats()).thenReturn(Set.of(CompositeTestHelper.stubFormat("lucene", 1, Set.of())));
when(registry.getIndexingEngine(any(), any())).thenAnswer(invocation -> {
DataFormatPlugin plugin = CompositeTestHelper.stubPlugin("lucene", 1);
return plugin.indexingEngine(null, null);
return plugin.indexingEngine(null);
});

Settings settings = Settings.builder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import org.opensearch.index.engine.exec.commit.Committer;
import org.opensearch.index.engine.exec.commit.IndexStoreProvider;
import org.opensearch.index.engine.exec.coord.CatalogSnapshot;
import org.opensearch.index.store.FormatChecksumStrategy;

import java.util.Collection;
import java.util.Collections;
Expand Down Expand Up @@ -72,7 +71,7 @@ static CompositeIndexingExecutionEngine createStubEngine(String primaryName, Str
when(registry.getIndexingEngine(any(), any())).thenAnswer(invocation -> {
DataFormat format = invocation.getArgument(1);
DataFormatPlugin plugin = plugins.get(format.name());
return plugin.indexingEngine(null, null);
return plugin.indexingEngine(null);
});

Settings.Builder settingsBuilder = Settings.builder()
Expand Down Expand Up @@ -101,7 +100,7 @@ public DataFormat getDataFormat() {
}

@Override
public IndexingExecutionEngine<?, ?> indexingEngine(IndexingEngineConfig settings, FormatChecksumStrategy checksumStrategy) {
public IndexingExecutionEngine<?, ?> indexingEngine(IndexingEngineConfig settings) {
return new StubIndexingExecutionEngine(format);
}
};
Expand All @@ -116,7 +115,7 @@ public DataFormat getDataFormat() {
}

@Override
public IndexingExecutionEngine<?, ?> indexingEngine(IndexingEngineConfig settings, FormatChecksumStrategy checksumStrategy) {
public IndexingExecutionEngine<?, ?> indexingEngine(IndexingEngineConfig settings) {
return new StubIndexingExecutionEngine(format);
}
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import org.opensearch.index.engine.dataformat.DataFormatRegistry;
import org.opensearch.index.engine.dataformat.IndexingEngineConfig;
import org.opensearch.index.engine.dataformat.IndexingExecutionEngine;
import org.opensearch.index.store.FormatChecksumStrategy;
import org.opensearch.index.store.PrecomputedChecksumStrategy;
import org.opensearch.parquet.engine.ParquetDataFormat;
import org.opensearch.parquet.engine.ParquetIndexingEngine;
Expand Down Expand Up @@ -52,10 +51,9 @@
* {@link #createComponents} and passes them to the per-shard
* {@link ParquetIndexingEngine} instances created in {@link #indexingEngine}.
*
* <p>The descriptor provides a {@link PrecomputedChecksumStrategy} that the directory
* holds at construction time. The {@link ParquetIndexingEngine} receives the same
* strategy instance from the directory via
* {@link org.opensearch.index.store.DataFormatAwareStoreDirectory#getChecksumStrategy},
* <p>The descriptor provides a {@link PrecomputedChecksumStrategy} that is created once
* per shard during initialization. The same strategy instance is shared between the
* directory and the {@link ParquetIndexingEngine} via the checksum strategies map,
* so pre-computed CRC32 values registered during write are directly visible to the
* upload path — no post-construction wiring needed.
*
Expand Down Expand Up @@ -99,15 +97,15 @@ public DataFormat getDataFormat() {
}

@Override
public IndexingExecutionEngine<?, ?> indexingEngine(IndexingEngineConfig engineConfig, FormatChecksumStrategy checksumStrategy) {
public IndexingExecutionEngine<?, ?> indexingEngine(IndexingEngineConfig engineConfig) {
return new ParquetIndexingEngine(
settings,
dataFormat,
engineConfig.store().shardPath(),
() -> ArrowSchemaBuilder.getSchema(engineConfig.mapperService()),
engineConfig.indexSettings(),
threadPool,
checksumStrategy
engineConfig.checksumStrategies().get(ParquetDataFormat.PARQUET_DATA_FORMAT_NAME)
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -734,6 +734,7 @@ public static final IndexShard newIndexShard(
clusterService.getClusterApplierService(),
MergedSegmentPublisher.EMPTY,
ReferencedSegmentsPublisher.EMPTY,
Collections.emptyMap(),
null // TODO
);
}
Expand Down
16 changes: 13 additions & 3 deletions server/src/main/java/org/opensearch/index/IndexService.java
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
import org.opensearch.index.similarity.SimilarityService;
import org.opensearch.index.store.DataFormatAwareStoreDirectory;
import org.opensearch.index.store.DataFormatAwareStoreDirectoryFactory;
import org.opensearch.index.store.FormatChecksumStrategy;
import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory;
import org.opensearch.index.store.Store;
import org.opensearch.index.store.remote.filecache.FileCache;
Expand Down Expand Up @@ -773,6 +774,10 @@ protected void closeInternal() {
}

Directory directory = null;
Map<String, FormatChecksumStrategy> checksumStrategies = Collections.emptyMap();
if (this.indexSettings.isPluggableDataFormatEnabled() && dataFormatRegistry != null) {
checksumStrategies = dataFormatRegistry.createChecksumStrategies(this.indexSettings);
}
if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_SETTING) &&
// TODO : Need to remove this check after support for hot indices is added in Composite Directory
this.indexSettings.isWarmIndex()) {
Expand All @@ -788,7 +793,7 @@ protected void closeInternal() {
directory = directoryFactory.newDirectory(this.indexSettings, path);
} else {
// Will be enabled in case of formatAware indices.
directory = createDataFormatAwareStoreDirectory(shardId, path);
directory = createDataFormatAwareStoreDirectory(shardId, path, checksumStrategies);
}
store = storeFactory.newStore(
shardId,
Expand Down Expand Up @@ -839,6 +844,7 @@ protected void closeInternal() {
clusterService.getClusterApplierService(),
this.indexSettings.isSegRepEnabledOrRemoteNode() ? mergedSegmentPublisher : null,
this.indexSettings.isSegRepEnabledOrRemoteNode() ? referencedSegmentsPublisher : null,
checksumStrategies,
dataFormatRegistry
);
eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
Expand Down Expand Up @@ -1344,15 +1350,19 @@ public boolean isForceExecution() {
* Creates DataFormatAwareStoreDirectory using the factory if available, otherwise fallback to Store's internal creation.
* This method centralizes the directory creation logic and enables plugin-based format discovery.
*/
private DataFormatAwareStoreDirectory createDataFormatAwareStoreDirectory(ShardId shardId, ShardPath shardPath) throws IOException {
private DataFormatAwareStoreDirectory createDataFormatAwareStoreDirectory(
ShardId shardId,
ShardPath shardPath,
Map<String, FormatChecksumStrategy> checksumStrategies
) throws IOException {
if (dataFormatAwareStoreDirectoryFactory != null) {
logger.debug("Using DataFormatAwareStoreDirectoryFactory to create directory for shard path: {}", shardPath);
return dataFormatAwareStoreDirectoryFactory.newDataFormatAwareStoreDirectory(
indexSettings,
shardId,
shardPath,
directoryFactory,
dataFormatRegistry
checksumStrategies
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,8 @@ public DataFormatAwareEngine(EngineConfig engineConfig) {
config().getMapperService(),
config().getIndexSettings(),
config().getStore(),
registry
registry,
config().getChecksumStrategies()
),
registry.format(config().getIndexSettings().pluggableDataFormat())
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
import org.opensearch.index.mapper.ParsedDocument;
import org.opensearch.index.merge.MergedSegmentTransferTracker;
import org.opensearch.index.seqno.RetentionLeases;
import org.opensearch.index.store.FormatChecksumStrategy;
import org.opensearch.index.store.Store;
import org.opensearch.index.translog.InternalTranslogFactory;
import org.opensearch.index.translog.TranslogConfig;
Expand All @@ -70,8 +71,10 @@
import org.opensearch.indices.IndexingMemoryController;
import org.opensearch.threadpool.ThreadPool;

import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.BooleanSupplier;
Expand Down Expand Up @@ -123,6 +126,7 @@ public final class EngineConfig {
private final DataFormatRegistry dataFormatRegistry;
private final MapperService mapperService;
private final CommitterFactory committerFactory;
private final Map<String, FormatChecksumStrategy> checksumStrategies;

/**
* A supplier of the outstanding retention leases. This is used during merged operations to determine which operations that have been
Expand Down Expand Up @@ -316,6 +320,7 @@ private EngineConfig(Builder builder) {
this.dataFormatRegistry = builder.dataFormatRegistry;
this.mapperService = builder.mapperService;
this.committerFactory = builder.committerFactory;
this.checksumStrategies = builder.checksumStrategies;
}

/**
Expand Down Expand Up @@ -655,6 +660,10 @@ public CommitterFactory getCommitterFactory() {
return this.committerFactory;
}

public Map<String, FormatChecksumStrategy> getChecksumStrategies() {
return this.checksumStrategies;
}

/**
* Builder for EngineConfig class
*
Expand Down Expand Up @@ -696,6 +705,7 @@ public static class Builder {
private DataFormatRegistry dataFormatRegistry;
private MapperService mapperService;
private CommitterFactory committerFactory;
private Map<String, FormatChecksumStrategy> checksumStrategies = Collections.emptyMap();

public Builder shardId(ShardId shardId) {
this.shardId = shardId;
Expand Down Expand Up @@ -867,6 +877,11 @@ public Builder committerFactory(CommitterFactory committerFactory) {
return this;
}

public Builder checksumStrategies(Map<String, FormatChecksumStrategy> checksumStrategies) {
this.checksumStrategies = checksumStrategies;
return this;
}

public EngineConfig build() {
return new EngineConfig(this);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.opensearch.index.mapper.MapperService;
import org.opensearch.index.merge.MergedSegmentTransferTracker;
import org.opensearch.index.seqno.RetentionLeases;
import org.opensearch.index.store.FormatChecksumStrategy;
import org.opensearch.index.store.Store;
import org.opensearch.index.translog.TranslogConfig;
import org.opensearch.index.translog.TranslogDeletionPolicyFactory;
Expand All @@ -47,6 +48,7 @@
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.BooleanSupplier;
import java.util.function.LongSupplier;
Expand Down Expand Up @@ -184,7 +186,8 @@ public EngineConfig newEngineConfig(
ClusterApplierService clusterApplierService,
MergedSegmentTransferTracker mergedSegmentTransferTracker,
DataFormatRegistry dataFormatRegistry,
MapperService mapperService
MapperService mapperService,
Map<String, FormatChecksumStrategy> checksumStrategies
) {
CodecService codecServiceToUse = codecService;
if (codecService == null && this.codecServiceFactory != null) {
Expand Down Expand Up @@ -225,6 +228,7 @@ public EngineConfig newEngineConfig(
.dataFormatRegistry(dataFormatRegistry)
.mapperService(mapperService)
.committerFactory(committerFactory)
.checksumStrategies(checksumStrategies)
.build();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
* <p>The checksum strategy here is the <em>default fallback</em> — a full-file scan.
* At runtime, the {@link IndexingExecutionEngine} may override this with a more
* efficient strategy (e.g., {@link org.opensearch.index.store.PrecomputedChecksumStrategy})
* via {@link org.opensearch.index.store.DataFormatAwareStoreDirectory#registerChecksumStrategy}.
* via the shared checksum strategies map created during shard initialization.
*
* @opensearch.experimental
*/
Expand Down
Loading
Loading