diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/clear_cache.yml b/x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/clear_cache.yml index 76c1bc33c328d..9081280aa4bc2 100644 --- a/x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/clear_cache.yml +++ b/x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/clear_cache.yml @@ -51,15 +51,6 @@ setup: indices.delete: index: docs - - do: - snapshot.create_repository: - repository: repository-searchable-snapshots - body: - type: searchable - settings: - delegate_type: fs - location: "repository-fs" - --- teardown: @@ -73,10 +64,6 @@ teardown: snapshot.delete_repository: repository: repository-fs - - do: - snapshot.delete_repository: - repository: repository-searchable-snapshots - --- "Clear searchable snapshots cache": - skip: @@ -116,10 +103,12 @@ teardown: - match: { error.root_cause.0.reason: "No searchable snapshots indices found" } - do: - snapshot.restore: - repository: repository-searchable-snapshots + searchable_snapshots.mount: + repository: repository-fs snapshot: snapshot wait_for_completion: true + body: + index: docs - match: { snapshot.snapshot: snapshot } - match: { snapshot.shards.failed: 0 } diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/stats.yml b/x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/stats.yml index 7f5a02f35e611..c7d2277ac6e9a 100644 --- a/x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/stats.yml +++ b/x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/stats.yml @@ -102,21 +102,12 @@ teardown: - match: { error.root_cause.0.reason: "No searchable snapshots indices found" } - do: - snapshot.create_repository: - repository: repository-searchable-snapshots - body: - type: searchable - settings: - delegate_type: fs - location: "repository-fs" - - - match: { acknowledged: true } - - - do: - snapshot.restore: - repository: repository-searchable-snapshots + searchable_snapshots.mount: + repository: repository-fs snapshot: snapshot wait_for_completion: true + body: + index: docs - match: { snapshot.snapshot: snapshot } - match: { snapshot.shards.failed: 0 } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/InMemoryNoOpCommitDirectory.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/InMemoryNoOpCommitDirectory.java similarity index 98% rename from x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/InMemoryNoOpCommitDirectory.java rename to x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/InMemoryNoOpCommitDirectory.java index 802c2a471dc7f..d082b0bf6526c 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/InMemoryNoOpCommitDirectory.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/InMemoryNoOpCommitDirectory.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.searchablesnapshots; +package org.elasticsearch.index.store; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java index 38936c0ee93aa..591417f32c833 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java @@ -5,6 +5,9 @@ */ package org.elasticsearch.index.store; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.store.BaseDirectory; import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; @@ -13,14 +16,36 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.SingleInstanceLockFactory; import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; +import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.repositories.IndexId; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.xpack.searchablesnapshots.cache.CacheDirectory; +import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.Path; import java.util.Collection; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.LongSupplier; + +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_REPOSITORY_SETTING; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING; /** * Implementation of {@link Directory} that exposes files from a snapshot as a Lucene directory. Because snapshot are immutable this @@ -38,7 +63,7 @@ public class SearchableSnapshotDirectory extends BaseDirectory { private final BlobStoreIndexShardSnapshot snapshot; private final BlobContainer blobContainer; - public SearchableSnapshotDirectory(final BlobStoreIndexShardSnapshot snapshot, final BlobContainer blobContainer) { + SearchableSnapshotDirectory(final BlobStoreIndexShardSnapshot snapshot, final BlobContainer blobContainer) { super(new SingleInstanceLockFactory()); this.snapshot = Objects.requireNonNull(snapshot); this.blobContainer = Objects.requireNonNull(blobContainer); @@ -121,4 +146,52 @@ public void rename(String source, String dest) { private static UnsupportedOperationException unsupportedException() { return new UnsupportedOperationException("Searchable snapshot directory does not support this operation"); } + + public static Directory create(RepositoriesService repositories, + CacheService cache, + IndexSettings indexSettings, + ShardPath shardPath, + LongSupplier currentTimeNanosSupplier) throws IOException { + + final Repository repository = repositories.repository(SNAPSHOT_REPOSITORY_SETTING.get(indexSettings.getSettings())); + if (repository instanceof BlobStoreRepository == false) { + throw new IllegalArgumentException("Repository [" + repository + "] is not searchable"); + } + final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository; + + final IndexId indexId = new IndexId(indexSettings.getIndex().getName(), SNAPSHOT_INDEX_ID_SETTING.get(indexSettings.getSettings())); + final BlobContainer blobContainer = blobStoreRepository.shardContainer(indexId, shardPath.getShardId().id()); + + final SnapshotId snapshotId = new SnapshotId(SNAPSHOT_SNAPSHOT_NAME_SETTING.get(indexSettings.getSettings()), + SNAPSHOT_SNAPSHOT_ID_SETTING.get(indexSettings.getSettings())); + final BlobStoreIndexShardSnapshot snapshot = blobStoreRepository.loadShardSnapshot(blobContainer, snapshotId); + + Directory directory = new SearchableSnapshotDirectory(snapshot, blobContainer); + if (SNAPSHOT_CACHE_ENABLED_SETTING.get(indexSettings.getSettings())) { + final Path cacheDir = shardPath.getDataPath().resolve("snapshots").resolve(snapshotId.getUUID()); + directory = new CacheDirectory(directory, cache, cacheDir, snapshotId, indexId, shardPath.getShardId(), + currentTimeNanosSupplier); + } + directory = new InMemoryNoOpCommitDirectory(directory); + + final IndexWriterConfig indexWriterConfig = new IndexWriterConfig(null) + .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) + .setMergePolicy(NoMergePolicy.INSTANCE); + + try (IndexWriter indexWriter = new IndexWriter(directory, indexWriterConfig)) { + final Map userData = new HashMap<>(); + indexWriter.getLiveCommitData().forEach(e -> userData.put(e.getKey(), e.getValue())); + + final String translogUUID = Translog.createEmptyTranslog(shardPath.resolveTranslog(), + Long.parseLong(userData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)), + shardPath.getShardId(), 0L); + + userData.put(Translog.TRANSLOG_UUID_KEY, translogUUID); + indexWriter.setLiveCommitData(userData.entrySet()); + indexWriter.commit(); + } + + return directory; + } + } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotRepository.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotRepository.java deleted file mode 100644 index 144de3e4fe96d..0000000000000 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotRepository.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.searchablesnapshots; - -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.store.Directory; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.RepositoryMetaData; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.blobstore.BlobContainer; -import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.seqno.SequenceNumbers; -import org.elasticsearch.index.shard.ShardPath; -import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; -import org.elasticsearch.index.store.SearchableSnapshotDirectory; -import org.elasticsearch.index.translog.Translog; -import org.elasticsearch.plugins.IndexStorePlugin; -import org.elasticsearch.repositories.FilterRepository; -import org.elasticsearch.repositories.IndexId; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.repositories.Repository; -import org.elasticsearch.repositories.blobstore.BlobStoreRepository; -import org.elasticsearch.snapshots.SnapshotId; -import org.elasticsearch.xpack.searchablesnapshots.cache.CacheDirectory; -import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; -import java.util.function.LongSupplier; -import java.util.function.Supplier; - -import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; - -/** - * A repository that wraps a {@link BlobStoreRepository} to add settings to the index metadata during a restore to identify the source - * snapshot and index in order to create a {@link SearchableSnapshotDirectory} (and corresponding empty translog) to search these shards - * without needing to fully restore them. - */ -public class SearchableSnapshotRepository extends FilterRepository { - - public static final String TYPE = "searchable"; - - public static final Setting SNAPSHOT_REPOSITORY_SETTING = - Setting.simpleString("index.store.snapshot.repository_name", Setting.Property.IndexScope, Setting.Property.PrivateIndex); - public static final Setting SNAPSHOT_SNAPSHOT_NAME_SETTING = - Setting.simpleString("index.store.snapshot.snapshot_name", Setting.Property.IndexScope, Setting.Property.PrivateIndex); - public static final Setting SNAPSHOT_SNAPSHOT_ID_SETTING = - Setting.simpleString("index.store.snapshot.snapshot_uuid", Setting.Property.IndexScope, Setting.Property.PrivateIndex); - public static final Setting SNAPSHOT_INDEX_ID_SETTING = - Setting.simpleString("index.store.snapshot.index_uuid", Setting.Property.IndexScope, Setting.Property.PrivateIndex); - public static final Setting SNAPSHOT_CACHE_ENABLED_SETTING = - Setting.boolSetting("index.store.snapshot.cache.enabled", true, Setting.Property.IndexScope); - - public static final String SNAPSHOT_DIRECTORY_FACTORY_KEY = "snapshot"; - - private static final Setting DELEGATE_TYPE - = new Setting<>("delegate_type", "", Function.identity(), Setting.Property.NodeScope); - - private final BlobStoreRepository blobStoreRepository; - - public SearchableSnapshotRepository(Repository in) { - super(in); - if (in instanceof BlobStoreRepository == false) { - throw new IllegalArgumentException("Repository [" + in + "] does not support searchable snapshots"); - } - blobStoreRepository = (BlobStoreRepository) in; - } - - private Directory makeDirectory(IndexSettings indexSettings, ShardPath shardPath, CacheService cacheService, - LongSupplier currentTimeNanosSupplier) throws IOException { - - IndexId indexId = new IndexId(indexSettings.getIndex().getName(), SNAPSHOT_INDEX_ID_SETTING.get(indexSettings.getSettings())); - BlobContainer blobContainer = blobStoreRepository.shardContainer(indexId, shardPath.getShardId().id()); - - SnapshotId snapshotId = new SnapshotId(SNAPSHOT_SNAPSHOT_NAME_SETTING.get(indexSettings.getSettings()), - SNAPSHOT_SNAPSHOT_ID_SETTING.get(indexSettings.getSettings())); - BlobStoreIndexShardSnapshot snapshot = blobStoreRepository.loadShardSnapshot(blobContainer, snapshotId); - - Directory directory = new SearchableSnapshotDirectory(snapshot, blobContainer); - if (SNAPSHOT_CACHE_ENABLED_SETTING.get(indexSettings.getSettings())) { - final Path cacheDir = shardPath.getDataPath().resolve("snapshots").resolve(snapshotId.getUUID()); - directory = new CacheDirectory(directory, cacheService, cacheDir, snapshotId, indexId, shardPath.getShardId(), - currentTimeNanosSupplier); - } - directory = new InMemoryNoOpCommitDirectory(directory); - - final IndexWriterConfig indexWriterConfig = new IndexWriterConfig(null) - .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) - .setMergePolicy(NoMergePolicy.INSTANCE); - - try (IndexWriter indexWriter = new IndexWriter(directory, indexWriterConfig)) { - final Map userData = new HashMap<>(); - indexWriter.getLiveCommitData().forEach(e -> userData.put(e.getKey(), e.getValue())); - - final String translogUUID = Translog.createEmptyTranslog(shardPath.resolveTranslog(), - Long.parseLong(userData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)), - shardPath.getShardId(), 0L); - - userData.put(Translog.TRANSLOG_UUID_KEY, translogUUID); - indexWriter.setLiveCommitData(userData.entrySet()); - indexWriter.commit(); - } - - return directory; - } - - @Override - public IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId index) throws IOException { - final IndexMetaData indexMetaData = super.getSnapshotIndexMetaData(snapshotId, index); - final IndexMetaData.Builder builder = IndexMetaData.builder(indexMetaData); - builder.settings(Settings.builder().put(indexMetaData.getSettings()).put(getIndexSettings(blobStoreRepository, snapshotId, index))); - return builder.build(); - } - - public static Settings getIndexSettings(Repository repository, SnapshotId snapshotId, IndexId indexId) { - return Settings.builder() - .put(SNAPSHOT_REPOSITORY_SETTING.getKey(), repository.getMetadata().name()) - .put(SNAPSHOT_SNAPSHOT_NAME_SETTING.getKey(), snapshotId.getName()) - .put(SNAPSHOT_SNAPSHOT_ID_SETTING.getKey(), snapshotId.getUUID()) - .put(SNAPSHOT_INDEX_ID_SETTING.getKey(), indexId.getId()) - .put(INDEX_STORE_TYPE_SETTING.getKey(), SNAPSHOT_DIRECTORY_FACTORY_KEY) - .put(IndexMetaData.SETTING_BLOCKS_WRITE, true) - .build(); - } - - static Factory getRepositoryFactory() { - return new Repository.Factory() { - @Override - public Repository create(RepositoryMetaData metadata) { - throw new UnsupportedOperationException(); - } - - @Override - public Repository create(RepositoryMetaData metaData, Function typeLookup) throws Exception { - String delegateType = DELEGATE_TYPE.get(metaData.settings()); - if (Strings.hasLength(delegateType) == false) { - throw new IllegalArgumentException(DELEGATE_TYPE.getKey() + " must be set"); - } - Repository.Factory factory = typeLookup.apply(delegateType); - return new SearchableSnapshotRepository(factory.create(new RepositoryMetaData(metaData.name(), - delegateType, metaData.settings()), typeLookup)); - } - }; - } - - public static IndexStorePlugin.DirectoryFactory newDirectoryFactory(final Supplier repositoriesService, - final Supplier cacheService, - final LongSupplier currentTimeNanosSupplier) { - return (indexSettings, shardPath) -> { - final RepositoriesService repositories = repositoriesService.get(); - assert repositories != null; - - final Repository repository = repositories.repository(SNAPSHOT_REPOSITORY_SETTING.get(indexSettings.getSettings())); - if (repository instanceof SearchableSnapshotRepository == false) { - throw new IllegalArgumentException("Repository [" + repository + "] is not searchable"); - } - - final CacheService cache = cacheService.get(); - assert cache != null; - - return ((SearchableSnapshotRepository) repository).makeDirectory(indexSettings, shardPath, cache, currentTimeNanosSupplier); - }; - } -} diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index 296c1fa96ca34..54ae2d535d0b4 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -24,6 +24,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.engine.ReadOnlyEngine; +import org.elasticsearch.index.store.SearchableSnapshotDirectory; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.EnginePlugin; @@ -32,22 +33,23 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesModule; import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.repositories.Repository; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xpack.searchablesnapshots.action.ClearSearchableSnapshotsCacheAction; +import org.elasticsearch.xpack.searchablesnapshots.action.MountSearchableSnapshotAction; import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsAction; import org.elasticsearch.xpack.searchablesnapshots.action.TransportClearSearchableSnapshotsCacheAction; +import org.elasticsearch.xpack.searchablesnapshots.action.TransportMountSearchableSnapshotAction; import org.elasticsearch.xpack.searchablesnapshots.action.TransportSearchableSnapshotsStatsAction; import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; import org.elasticsearch.xpack.searchablesnapshots.rest.RestClearSearchableSnapshotsCacheAction; +import org.elasticsearch.xpack.searchablesnapshots.rest.RestMountSearchableSnapshotAction; import org.elasticsearch.xpack.searchablesnapshots.rest.RestSearchableSnapshotsStatsAction; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -61,6 +63,19 @@ */ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, RepositoryPlugin, EnginePlugin, ActionPlugin { + public static final Setting SNAPSHOT_REPOSITORY_SETTING = + Setting.simpleString("index.store.snapshot.repository_name", Setting.Property.IndexScope, Setting.Property.PrivateIndex); + public static final Setting SNAPSHOT_SNAPSHOT_NAME_SETTING = + Setting.simpleString("index.store.snapshot.snapshot_name", Setting.Property.IndexScope, Setting.Property.PrivateIndex); + public static final Setting SNAPSHOT_SNAPSHOT_ID_SETTING = + Setting.simpleString("index.store.snapshot.snapshot_uuid", Setting.Property.IndexScope, Setting.Property.PrivateIndex); + public static final Setting SNAPSHOT_INDEX_ID_SETTING = + Setting.simpleString("index.store.snapshot.index_uuid", Setting.Property.IndexScope, Setting.Property.PrivateIndex); + public static final Setting SNAPSHOT_CACHE_ENABLED_SETTING = + Setting.boolSetting("index.store.snapshot.cache.enabled", true, Setting.Property.IndexScope); + + public static final String SNAPSHOT_DIRECTORY_FACTORY_KEY = "snapshot"; + private final SetOnce repositoriesService; private final SetOnce cacheService; private final Settings settings; @@ -73,11 +88,11 @@ public SearchableSnapshots(final Settings settings) { @Override public List> getSettings() { - return List.of(SearchableSnapshotRepository.SNAPSHOT_REPOSITORY_SETTING, - SearchableSnapshotRepository.SNAPSHOT_SNAPSHOT_NAME_SETTING, - SearchableSnapshotRepository.SNAPSHOT_SNAPSHOT_ID_SETTING, - SearchableSnapshotRepository.SNAPSHOT_INDEX_ID_SETTING, - SearchableSnapshotRepository.SNAPSHOT_CACHE_ENABLED_SETTING, + return List.of(SNAPSHOT_REPOSITORY_SETTING, + SNAPSHOT_SNAPSHOT_NAME_SETTING, + SNAPSHOT_SNAPSHOT_ID_SETTING, + SNAPSHOT_INDEX_ID_SETTING, + SNAPSHOT_CACHE_ENABLED_SETTING, CacheService.SNAPSHOT_CACHE_SIZE_SETTING, CacheService.SNAPSHOT_CACHE_RANGE_SIZE_SETTING ); @@ -103,35 +118,36 @@ public Collection createComponents( @Override public void onRepositoriesModule(RepositoriesModule repositoriesModule) { - repositoriesService.set(repositoriesModule.getRepositoryService()); // should we use some SPI mechanism? + // TODO NORELEASE should we use some SPI mechanism? The only reason we are a RepositoriesPlugin is because of this :/ + repositoriesService.set(repositoriesModule.getRepositoryService()); } @Override public Map getDirectoryFactories() { - return Map.of(SearchableSnapshotRepository.SNAPSHOT_DIRECTORY_FACTORY_KEY, - SearchableSnapshotRepository.newDirectoryFactory(repositoriesService::get, cacheService::get, System::nanoTime)); + return Map.of(SNAPSHOT_DIRECTORY_FACTORY_KEY, (indexSettings, shardPath) -> { + final RepositoriesService repositories = repositoriesService.get(); + assert repositories != null; + final CacheService cache = cacheService.get(); + assert cache != null; + return SearchableSnapshotDirectory.create(repositories, cache, indexSettings, shardPath, System::nanoTime); + }); } @Override public Optional getEngineFactory(IndexSettings indexSettings) { - if (SearchableSnapshotRepository.SNAPSHOT_DIRECTORY_FACTORY_KEY.equals(INDEX_STORE_TYPE_SETTING.get(indexSettings.getSettings())) + if (SNAPSHOT_DIRECTORY_FACTORY_KEY.equals(INDEX_STORE_TYPE_SETTING.get(indexSettings.getSettings())) && indexSettings.getSettings().getAsBoolean("index.frozen", false) == false) { return Optional.of(engineConfig -> new ReadOnlyEngine(engineConfig, null, new TranslogStats(), false, Function.identity())); } return Optional.empty(); } - @Override - public Map getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry, - ClusterService clusterService) { - return Collections.singletonMap(SearchableSnapshotRepository.TYPE, SearchableSnapshotRepository.getRepositoryFactory()); - } - @Override public List> getActions() { return List.of( new ActionHandler<>(SearchableSnapshotsStatsAction.INSTANCE, TransportSearchableSnapshotsStatsAction.class), - new ActionHandler<>(ClearSearchableSnapshotsCacheAction.INSTANCE, TransportClearSearchableSnapshotsCacheAction.class) + new ActionHandler<>(ClearSearchableSnapshotsCacheAction.INSTANCE, TransportClearSearchableSnapshotsCacheAction.class), + new ActionHandler<>(MountSearchableSnapshotAction.INSTANCE, TransportMountSearchableSnapshotAction.class) ); } @@ -141,8 +157,10 @@ public List getRestHandlers(Settings settings, RestController restC Supplier nodesInCluster) { return List.of( new RestSearchableSnapshotsStatsAction(), - new RestClearSearchableSnapshotsCacheAction() + new RestClearSearchableSnapshotsCacheAction(), + new RestMountSearchableSnapshotAction() ); } + } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/AbstractTransportSearchableSnapshotsAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/AbstractTransportSearchableSnapshotsAction.java index e9d5b9b20501d..6ceefbba1d9a1 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/AbstractTransportSearchableSnapshotsAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/AbstractTransportSearchableSnapshotsAction.java @@ -24,9 +24,9 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.store.InMemoryNoOpCommitDirectory; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.searchablesnapshots.InMemoryNoOpCommitDirectory; import org.elasticsearch.xpack.searchablesnapshots.cache.CacheDirectory; import java.io.IOException; @@ -34,8 +34,8 @@ import java.util.List; import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; -import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotRepository.SNAPSHOT_CACHE_ENABLED_SETTING; -import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotRepository.SNAPSHOT_DIRECTORY_FACTORY_KEY; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_DIRECTORY_FACTORY_KEY; public abstract class AbstractTransportSearchableSnapshotsAction , Response extends BroadcastResponse, ShardOperationResult extends Writeable> diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotAction.java new file mode 100644 index 0000000000000..49e8e38900c59 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotAction.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchablesnapshots.action; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; + +public class MountSearchableSnapshotAction extends ActionType { + + public static final MountSearchableSnapshotAction INSTANCE = new MountSearchableSnapshotAction(); + public static final String NAME = "cluster:admin/snapshot/mount"; + + private MountSearchableSnapshotAction() { + super(NAME, RestoreSnapshotResponse::new); + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequest.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequest.java new file mode 100644 index 0000000000000..7abc1d47ed54f --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequest.java @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchablesnapshots.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestRequest; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; +import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class MountSearchableSnapshotRequest extends MasterNodeRequest { + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "mount_searchable_snapshot", true, + (a, request) -> new MountSearchableSnapshotRequest( + Objects.requireNonNullElse((String)a[1], (String)a[0]), + request.param("repository"), + request.param("snapshot"), + (String)a[0], + Objects.requireNonNullElse((Settings)a[2], Settings.EMPTY), + Objects.requireNonNullElse((String[])a[3], Strings.EMPTY_ARRAY), + request.paramAsBoolean("wait_for_completion", false))); + + private static final ParseField INDEX_FIELD = new ParseField("index"); + private static final ParseField RENAMED_INDEX_FIELD = new ParseField("renamed_index"); + private static final ParseField INDEX_SETTINGS_FIELD = new ParseField("index_settings"); + private static final ParseField IGNORE_INDEX_SETTINGS_FIELD = new ParseField("ignore_index_settings"); + + static { + PARSER.declareField(constructorArg(), XContentParser::text, INDEX_FIELD, ObjectParser.ValueType.STRING); + PARSER.declareField(optionalConstructorArg(), XContentParser::text, RENAMED_INDEX_FIELD, ObjectParser.ValueType.STRING); + PARSER.declareField(optionalConstructorArg(), Settings::fromXContent, INDEX_SETTINGS_FIELD, ObjectParser.ValueType.OBJECT); + PARSER.declareField(optionalConstructorArg(), + p -> p.list().stream().map(s -> (String) s).collect(Collectors.toList()).toArray(Strings.EMPTY_ARRAY), + IGNORE_INDEX_SETTINGS_FIELD, ObjectParser.ValueType.STRING_ARRAY); + } + + private final String mountedIndexName; + private final String repositoryName; + private final String snapshotName; + private final String snapshotIndexName; + private final Settings indexSettings; + private final String[] ignoredIndexSettings; + private final boolean waitForCompletion; + + /** + * Constructs a new mount searchable snapshot request, restoring an index with the settings needed to make it a searchable snapshot. + */ + public MountSearchableSnapshotRequest(String mountedIndexName, String repositoryName, String snapshotName, String snapshotIndexName, + Settings indexSettings, String[] ignoredIndexSettings, boolean waitForCompletion) { + this.mountedIndexName = Objects.requireNonNull(mountedIndexName); + this.repositoryName = Objects.requireNonNull(repositoryName); + this.snapshotName = Objects.requireNonNull(snapshotName); + this.snapshotIndexName = Objects.requireNonNull(snapshotIndexName); + this.indexSettings = Objects.requireNonNull(indexSettings); + this.ignoredIndexSettings = Objects.requireNonNull(ignoredIndexSettings); + this.waitForCompletion = waitForCompletion; + } + + MountSearchableSnapshotRequest(StreamInput in) throws IOException { + super(in); + this.mountedIndexName = in.readString(); + this.repositoryName = in.readString(); + this.snapshotName = in.readString(); + this.snapshotIndexName = in.readString(); + this.indexSettings = readSettingsFromStream(in); + this.ignoredIndexSettings = in.readStringArray(); + this.waitForCompletion = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(mountedIndexName); + out.writeString(repositoryName); + out.writeString(snapshotName); + out.writeString(snapshotIndexName); + writeSettingsToStream(indexSettings, out); + out.writeStringArray(ignoredIndexSettings); + out.writeBoolean(waitForCompletion); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + /** + * @return the name of the index that will be created + */ + public String mountedIndexName() { + return mountedIndexName; + } + + /** + * @return the name of the repository + */ + public String repositoryName() { + return this.repositoryName; + } + + /** + * @return the name of the snapshot. + */ + public String snapshotName() { + return this.snapshotName; + } + + /** + * @return the name of the index contained in the snapshot + */ + public String snapshotIndexName() { + return snapshotIndexName; + } + + /** + * @return true if the operation will wait for completion + */ + public boolean waitForCompletion() { + return waitForCompletion; + } + + /** + * @return settings that should be added to the index when it is mounted + */ + public Settings indexSettings() { + return this.indexSettings; + } + + /** + * @return the names of settings that should be removed from the index when it is mounted + */ + public String[] ignoreIndexSettings() { + return ignoredIndexSettings; + } + + @Override + public String getDescription() { + return "mount snapshot [" + repositoryName + ":" + snapshotName + ":" + snapshotIndexName + "] as [" + mountedIndexName + "]"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MountSearchableSnapshotRequest that = (MountSearchableSnapshotRequest) o; + return waitForCompletion == that.waitForCompletion && + Objects.equals(mountedIndexName, that.mountedIndexName) && + Objects.equals(repositoryName, that.repositoryName) && + Objects.equals(snapshotName, that.snapshotName) && + Objects.equals(snapshotIndexName, that.snapshotIndexName) && + Objects.equals(indexSettings, that.indexSettings) && + Arrays.equals(ignoredIndexSettings, that.ignoredIndexSettings) && + Objects.equals(masterNodeTimeout, that.masterNodeTimeout); + } + + @Override + public int hashCode() { + int result = Objects.hash(mountedIndexName, repositoryName, snapshotName, snapshotIndexName, indexSettings, waitForCompletion, + masterNodeTimeout); + result = 31 * result + Arrays.hashCode(ignoredIndexSettings); + return result; + } + + @Override + public String toString() { + return getDescription(); + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java new file mode 100644 index 0000000000000..ec9ada75c0fc6 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchablesnapshots.action; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.StepListener; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.repositories.IndexId; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots; + +import java.io.IOException; +import java.util.Map; +import java.util.Optional; + +import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; + +/** + * Action that mounts a snapshot as a searchable snapshot, by converting the mount request into a restore request with specific settings + * using {@link TransportMountSearchableSnapshotAction#buildIndexSettings(String, SnapshotId, IndexId)}. + * + * This action doesn't technically need to run on the master node, but it needs to get metadata from the repository and we only expect the + * repository to be accessible from data and master-eligible nodes so we can't run it everywhere. Given that we already have a way to run + * actions on the master and that we have to do the restore via the master, it's simplest to use {@link TransportMasterNodeAction}. + */ +public class TransportMountSearchableSnapshotAction + extends TransportMasterNodeAction { + + private final Client client; + private final RepositoriesService repositoriesService; + + @Inject + public TransportMountSearchableSnapshotAction(TransportService transportService, ClusterService clusterService, Client client, + ThreadPool threadPool, RepositoriesService repositoriesService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(MountSearchableSnapshotAction.NAME, transportService, clusterService, threadPool, actionFilters, + MountSearchableSnapshotRequest::new, indexNameExpressionResolver); + this.client = client; + this.repositoriesService = repositoriesService; + } + + @Override + protected String executor() { + // Avoid SNAPSHOT since snapshot threads may all be busy with long-running tasks which would block this action from responding with + // an error. Avoid SAME since getting the repository metadata may block on IO. + return ThreadPool.Names.GENERIC; + } + + @Override + protected RestoreSnapshotResponse read(StreamInput in) throws IOException { + return new RestoreSnapshotResponse(in); + } + + @Override + protected ClusterBlockException checkBlock(MountSearchableSnapshotRequest request, ClusterState state) { + // The restore action checks the cluster blocks. + return null; + } + + /** + * Return the index settings required to make a snapshot searchable + */ + private static Settings buildIndexSettings(String repoName, SnapshotId snapshotId, IndexId indexId) { + return Settings.builder() + .put(SearchableSnapshots.SNAPSHOT_REPOSITORY_SETTING.getKey(), repoName) + .put(SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING.getKey(), snapshotId.getName()) + .put(SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING.getKey(), snapshotId.getUUID()) + .put(SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING.getKey(), indexId.getId()) + .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshots.SNAPSHOT_DIRECTORY_FACTORY_KEY) + .put(IndexMetaData.SETTING_BLOCKS_WRITE, true) + .build(); + } + + @Override + protected void masterOperation(Task task, final MountSearchableSnapshotRequest request, final ClusterState state, + final ActionListener listener) { + + final String repoName = request.repositoryName(); + final String snapName = request.snapshotName(); + final String indexName = request.snapshotIndexName(); + + // Retrieve IndexId and SnapshotId instances, which are then used to create a new restore + // request, which is then sent on to the actual snapshot restore mechanism + final Repository repository = repositoriesService.repository(repoName); + final StepListener repositoryDataListener = new StepListener<>(); + repository.getRepositoryData(repositoryDataListener); + repositoryDataListener.whenComplete(repoData -> { + final Map indexIds = repoData.getIndices(); + if (indexIds.containsKey(indexName) == false) { + throw new IndexNotFoundException("index [" + indexName + "] not found in repository [" + repoName + "]"); + } + final IndexId indexId = indexIds.get(indexName); + + final Optional matchingSnapshotId = repoData.getSnapshotIds().stream() + .filter(s -> snapName.equals(s.getName())).findFirst(); + if (matchingSnapshotId.isEmpty()) { + throw new ElasticsearchException("snapshot [" + snapName + "] not found in repository [" + repoName + "]"); + } + final SnapshotId snapshotId = matchingSnapshotId.get(); + + // We must fail the restore if it obtains different IDs from the ones we just obtained (e.g. the target snapshot was replaced + // by one with the same name while we are restoring it) or else the index metadata might bear no relation to the snapshot we're + // searching. TODO NORELEASE validate IDs in the restore. + + client.admin().cluster().restoreSnapshot(new RestoreSnapshotRequest(repoName, snapName) + // Restore the single index specified + .indices(indexName) + // Always rename it to the desired mounted index name + .renamePattern(".+") + .renameReplacement(request.mountedIndexName()) + // Pass through index settings, adding the index-level settings required to use searchable snapshots + .indexSettings(Settings.builder().put(request.indexSettings()) + .put(buildIndexSettings(request.repositoryName(), snapshotId, indexId)) + .build()) + // Pass through ignored index settings + .ignoreIndexSettings(request.ignoreIndexSettings()) + // Don't include global state + .includeGlobalState(false) + // Don't include aliases + .includeAliases(false) + // Pass through the wait-for-completion flag + .waitForCompletion(request.waitForCompletion()) + // Pass through the master-node timeout + .masterNodeTimeout(request.masterNodeTimeout()), listener); + }, listener::onFailure); + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/rest/RestMountSearchableSnapshotAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/rest/RestMountSearchableSnapshotAction.java new file mode 100644 index 0000000000000..c0d1029348b22 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/rest/RestMountSearchableSnapshotAction.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchablesnapshots.rest; + +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.searchablesnapshots.action.MountSearchableSnapshotAction; +import org.elasticsearch.xpack.searchablesnapshots.action.MountSearchableSnapshotRequest; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.POST; + +public class RestMountSearchableSnapshotAction extends BaseRestHandler { + @Override + public String getName() { + return "mount_snapshot_action"; + } + + @Override + public List routes() { + return Collections.singletonList(new Route(POST, "/_snapshot/{repository}/{snapshot}/_mount")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + MountSearchableSnapshotRequest mountSearchableSnapshotRequest + = MountSearchableSnapshotRequest.PARSER.apply(request.contentParser(), request) + .masterNodeTimeout(request.paramAsTime("master_timeout", MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT)); + return channel -> client.execute(MountSearchableSnapshotAction.INSTANCE, mountSearchableSnapshotRequest, + new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/InMemoryNoOpCommitDirectoryTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/InMemoryNoOpCommitDirectoryTests.java similarity index 99% rename from x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/InMemoryNoOpCommitDirectoryTests.java rename to x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/InMemoryNoOpCommitDirectoryTests.java index 09e12a6147533..de184116e0116 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/InMemoryNoOpCommitDirectoryTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/InMemoryNoOpCommitDirectoryTests.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.searchablesnapshots; +package org.elasticsearch.index.store; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java index 10e9d429c61bb..11fdd5c34daab 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java @@ -65,7 +65,6 @@ import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotRepository; import java.io.Closeable; import java.io.EOFException; @@ -324,7 +323,7 @@ private void testDirectories(final CheckedBiConsumer restoring index [{}] with cache [{}]", restoredIndexName, cacheEnabled ? "enabled" : "disabled"); - final RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster() - .prepareRestoreSnapshot(searchableRepoName, snapshotName).setIndices(indexName) - .setRenamePattern(indexName) - .setRenameReplacement(restoredIndexName) - .setIndexSettings(Settings.builder() - .put(SearchableSnapshotRepository.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), cacheEnabled) + final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(restoredIndexName, fsRepoName, + snapshotInfo.snapshotId().getName(), indexName, + Settings.builder() + .put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), cacheEnabled) .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), Boolean.FALSE.toString()) - .build()) - .setWaitForCompletion(true).get(); + .build(), Strings.EMPTY_ARRAY, true); + + final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, req).get(); assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0)); final Settings settings = client().admin().indices().prepareGetSettings(restoredIndexName).get().getIndexToSettings().get(restoredIndexName); - assertThat(SearchableSnapshotRepository.SNAPSHOT_REPOSITORY_SETTING.get(settings), equalTo(searchableRepoName)); - assertThat(SearchableSnapshotRepository.SNAPSHOT_SNAPSHOT_NAME_SETTING.get(settings), equalTo(snapshotName)); + assertThat(SearchableSnapshots.SNAPSHOT_REPOSITORY_SETTING.get(settings), equalTo(fsRepoName)); + assertThat(SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING.get(settings), equalTo(snapshotName)); assertThat(IndexModule.INDEX_STORE_TYPE_SETTING.get(settings), equalTo(SNAPSHOT_DIRECTORY_FACTORY_KEY)); assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(settings)); - assertTrue(SearchableSnapshotRepository.SNAPSHOT_SNAPSHOT_ID_SETTING.exists(settings)); - assertTrue(SearchableSnapshotRepository.SNAPSHOT_INDEX_ID_SETTING.exists(settings)); + assertTrue(SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING.exists(settings)); + assertTrue(SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING.exists(settings)); assertRecovered(restoredIndexName, originalAllHits, originalBarHits); diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequestTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequestTests.java new file mode 100644 index 0000000000000..6d1edae182825 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequestTests.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchablesnapshots.action; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.util.Arrays; + +public class MountSearchableSnapshotRequestTests extends AbstractWireSerializingTestCase { + + private MountSearchableSnapshotRequest randomState(MountSearchableSnapshotRequest instance) { + return new MountSearchableSnapshotRequest( + randomBoolean() ? instance.mountedIndexName() : mutateString(instance.mountedIndexName()), + randomBoolean() ? instance.repositoryName() : mutateString(instance.repositoryName()), + randomBoolean() ? instance.snapshotName() : mutateString(instance.snapshotName()), + randomBoolean() ? instance.snapshotIndexName() : mutateString(instance.snapshotIndexName()), + randomBoolean() ? instance.indexSettings() : mutateSettings(instance.indexSettings()), + randomBoolean() ? instance.ignoreIndexSettings() : mutateStringArray(instance.ignoreIndexSettings()), + randomBoolean()) + .masterNodeTimeout(randomBoolean() ? instance.masterNodeTimeout() : mutateTimeValue(instance.masterNodeTimeout())); + } + + @Override + protected MountSearchableSnapshotRequest createTestInstance() { + return randomState(new MountSearchableSnapshotRequest(randomAlphaOfLength(5), randomAlphaOfLength(5), randomAlphaOfLength(5), + randomAlphaOfLength(5), Settings.EMPTY, Strings.EMPTY_ARRAY, randomBoolean())); + } + + @Override + protected Writeable.Reader instanceReader() { + return MountSearchableSnapshotRequest::new; + } + + @Override + protected MountSearchableSnapshotRequest mutateInstance(MountSearchableSnapshotRequest req) { + switch (randomInt(7)) { + case 0: + return new MountSearchableSnapshotRequest(mutateString(req.mountedIndexName()), req.repositoryName(), req.snapshotName(), + req.snapshotIndexName(), req.indexSettings(), req.ignoreIndexSettings(), + req.waitForCompletion()).masterNodeTimeout(req.masterNodeTimeout()); + case 1: + return new MountSearchableSnapshotRequest(req.mountedIndexName(), mutateString(req.repositoryName()), req.snapshotName(), + req.snapshotIndexName(), req.indexSettings(), req.ignoreIndexSettings(), + req.waitForCompletion()).masterNodeTimeout(req.masterNodeTimeout()); + case 2: + return new MountSearchableSnapshotRequest(req.mountedIndexName(), req.repositoryName(), mutateString(req.snapshotName()), + req.snapshotIndexName(), req.indexSettings(), req.ignoreIndexSettings(), + req.waitForCompletion()).masterNodeTimeout(req.masterNodeTimeout()); + case 3: + return new MountSearchableSnapshotRequest(req.mountedIndexName(), req.repositoryName(), req.snapshotName(), + mutateString(req.snapshotIndexName()), req.indexSettings(), req.ignoreIndexSettings(), + req.waitForCompletion()).masterNodeTimeout(req.masterNodeTimeout()); + case 4: + return new MountSearchableSnapshotRequest(req.mountedIndexName(), req.repositoryName(), req.snapshotName(), + req.snapshotIndexName(), mutateSettings(req.indexSettings()), req.ignoreIndexSettings(), + req.waitForCompletion()).masterNodeTimeout(req.masterNodeTimeout()); + case 5: + return new MountSearchableSnapshotRequest(req.mountedIndexName(), req.repositoryName(), req.snapshotName(), + req.snapshotIndexName(), req.indexSettings(), mutateStringArray(req.ignoreIndexSettings()), + req.waitForCompletion()).masterNodeTimeout(req.masterNodeTimeout()); + case 6: + return new MountSearchableSnapshotRequest(req.mountedIndexName(), req.repositoryName(), req.snapshotName(), + req.snapshotIndexName(), req.indexSettings(), req.ignoreIndexSettings(), + req.waitForCompletion() == false).masterNodeTimeout(req.masterNodeTimeout()); + + default: + return new MountSearchableSnapshotRequest(req.mountedIndexName(), req.repositoryName(), req.snapshotName(), + req.snapshotIndexName(), req.indexSettings(), req.ignoreIndexSettings(), + req.waitForCompletion()).masterNodeTimeout(mutateTimeValue(req.masterNodeTimeout())); + } + } + + private static TimeValue mutateTimeValue(TimeValue timeValue) { + long millis = timeValue.millis(); + long newMillis = randomValueOtherThan(millis, () -> randomLongBetween(0, 60000)); + return TimeValue.timeValueMillis(newMillis); + } + + private static String mutateString(String string) { + return randomAlphaOfLength(11 - string.length()); + } + + private static Settings mutateSettings(Settings settings) { + if (settings.size() < 5 && (settings.isEmpty() || randomBoolean())) { + return Settings.builder().put(settings).put(randomAlphaOfLength(3), randomAlphaOfLength(3)).build(); + } else { + return Settings.EMPTY; + } + } + + private static String[] mutateStringArray(String[] strings) { + if (strings.length < 5 && (strings.length == 0 || randomBoolean())) { + String[] newStrings = Arrays.copyOf(strings, strings.length + 1); + newStrings[strings.length] = randomAlphaOfLength(3); + return newStrings; + } else if (randomBoolean()) { + String[] newStrings = Arrays.copyOf(strings, strings.length); + int i = randomIntBetween(0, newStrings.length - 1); + newStrings[i] = mutateString(newStrings[i]); + return newStrings; + } else { + return Strings.EMPTY_ARRAY; + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/searchable_snapshots.mount.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/searchable_snapshots.mount.json new file mode 100644 index 0000000000000..4a9a0ad4c3235 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/searchable_snapshots.mount.json @@ -0,0 +1,43 @@ +{ + "searchable_snapshots.mount": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/searchable-snapshots-mount.html //NORELEASE This API should be documented. We expect this API to be stable at the time it is merged in master, but in case it is not its stability should be documented appropriately." + }, + "stability": "experimental", + "url": { + "paths": [ + { + "path": "/_snapshot/{repository}/{snapshot}/_mount", + "methods": [ + "POST" + ], + "parts": { + "repository": { + "type": "string", + "description": "The name of the repository containing the snapshot of the index to mount" + }, + "snapshot": { + "type": "string", + "description": "The name of the snapshot of the index to mount" + } + } + } + ] + }, + "params": { + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "wait_for_completion":{ + "type":"boolean", + "description":"Should this request wait until the operation has completed before returning", + "default":false + } + }, + "body":{ + "description":"The restore configuration for mounting the snapshot as searchable", + "required":true + } + } +}