diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 9aa7b65330699..4209ae1128d88 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -2196,6 +2196,12 @@ public static Index resolveIndex(String index) { return new Index(index, uuid); } + public static String resolveCustomDataPath(String index) { + GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(index).get(); + assertTrue("index " + index + " not found", getIndexResponse.getSettings().containsKey(index)); + return getIndexResponse.getSettings().get(index).get(IndexMetadata.SETTING_DATA_PATH); + } + public static boolean inFipsJvm() { return Boolean.parseBoolean(System.getProperty(FIPS_SYSPROP)); } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java index ffd87297a1117..0ba394f06e541 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java @@ -53,6 +53,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; @@ -113,6 +114,7 @@ public class SearchableSnapshotDirectory extends BaseDirectory { private final Set excludedFileTypes; private final long uncachedChunkSize; // if negative use BlobContainer#readBlobPreferredLength, see #getUncachedChunkSize() private final Path cacheDir; + private final ShardPath shardPath; private final AtomicBoolean closed; // volatile fields are updated once under `this` lock, all together, iff loaded is not true. @@ -130,6 +132,7 @@ public SearchableSnapshotDirectory( LongSupplier currentTimeNanosSupplier, CacheService cacheService, Path cacheDir, + ShardPath shardPath, ThreadPool threadPool ) { super(new SingleInstanceLockFactory()); @@ -142,6 +145,7 @@ public SearchableSnapshotDirectory( this.statsCurrentTimeNanosSupplier = Objects.requireNonNull(currentTimeNanosSupplier); this.cacheService = Objects.requireNonNull(cacheService); this.cacheDir = Objects.requireNonNull(cacheDir); + this.shardPath = Objects.requireNonNull(shardPath); this.closed = new AtomicBoolean(false); this.useCache = SNAPSHOT_CACHE_ENABLED_SETTING.get(indexSettings); this.prewarmCache = useCache ? SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING.get(indexSettings) : false; @@ -182,6 +186,7 @@ public boolean loadSnapshot() { this.blobContainer = blobContainerSupplier.get(); this.snapshot = snapshotSupplier.get(); this.loaded = true; + cleanExistingRegularShardFiles(); prewarmCache(); } } @@ -374,6 +379,14 @@ public String toString() { return this.getClass().getSimpleName() + "@snapshotId=" + snapshotId + " lockFactory=" + lockFactory; } + private void cleanExistingRegularShardFiles() { + try { + IOUtils.rm(shardPath.resolveIndex(), shardPath.resolveTranslog()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + private void prewarmCache() { if (prewarmCache) { final BlockingQueue, CheckedRunnable>> queue = new LinkedBlockingQueue<>(); @@ -513,6 +526,7 @@ public static Directory create( currentTimeNanosSupplier, cache, cacheDir, + shardPath, threadPool ) ); diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java index e41c95ba7f453..7e5b1e7613695 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java @@ -16,7 +16,9 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import org.elasticsearch.index.store.cache.TestUtils; @@ -28,7 +30,9 @@ import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongSupplier; @@ -597,6 +601,14 @@ private static void executeTestCase( final StoreFileMetadata metadata = new StoreFileMetadata(fileName, fileContent.length, "_checksum", Version.CURRENT.luceneVersion); final List files = List.of(new FileInfo(blobName, metadata, new ByteSizeValue(fileContent.length))); final BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getName(), 0L, files, 0L, 0L, 0, 0L); + final Path shardDir; + try { + shardDir = new NodeEnvironment.NodePath(createTempDir()).resolve(shardId); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId); + final Path cacheDir = createTempDir(); try ( CacheService ignored = cacheService; @@ -609,7 +621,8 @@ private static void executeTestCase( indexSettings, statsCurrentTimeNanos, cacheService, - createTempDir(), + cacheDir, + shardPath, threadPool ) { @Override diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java index a93d25ef6633d..519233f26915b 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java @@ -58,10 +58,12 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; import org.elasticsearch.index.store.cache.TestUtils; @@ -86,6 +88,7 @@ import java.io.EOFException; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.Files; @@ -548,6 +551,13 @@ protected void assertSnapshotOrGenericThread() { final BlobContainer blobContainer = repository.shardContainer(indexId, shardId.id()); final BlobStoreIndexShardSnapshot snapshot = repository.loadShardSnapshot(blobContainer, snapshotId); + final Path shardDir; + try { + shardDir = new NodeEnvironment.NodePath(createTempDir()).resolve(shardId); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId); final Path cacheDir = createTempDir(); final CacheService cacheService = TestUtils.createDefaultCacheService(); releasables.add(cacheService); @@ -567,6 +577,7 @@ protected void assertSnapshotOrGenericThread() { () -> 0L, cacheService, cacheDir, + shardPath, threadPool ) ) { @@ -637,6 +648,13 @@ public void testClearCache() throws Exception { final IndexId indexId = new IndexId("_id", "_uuid"); final ShardId shardId = new ShardId(new Index("_name", "_id"), 0); + final Path shardDir; + try { + shardDir = new NodeEnvironment.NodePath(createTempDir()).resolve(shardId); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId); final Path cacheDir = createTempDir(); final ThreadPool threadPool = new TestThreadPool(getTestName(), SearchableSnapshots.executorBuilders()); try ( @@ -655,6 +673,7 @@ public void testClearCache() throws Exception { () -> 0L, cacheService, cacheDir, + shardPath, threadPool ) ) { diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CachedBlobContainerIndexInputTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CachedBlobContainerIndexInputTests.java index 5532dbaecbb0d..08cc9bd8fd2fc 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CachedBlobContainerIndexInputTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CachedBlobContainerIndexInputTests.java @@ -12,7 +12,9 @@ import org.elasticsearch.common.lucene.store.ESIndexInputTestCase; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; import org.elasticsearch.index.store.SearchableSnapshotDirectory; import org.elasticsearch.index.store.StoreFileMetadata; @@ -27,6 +29,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.HashSet; @@ -83,6 +86,13 @@ public void testRandomReads() throws IOException { blobContainer = singleBlobContainer; } + final Path shardDir; + try { + shardDir = new NodeEnvironment.NodePath(createTempDir()).resolve(shardId); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId); final Path cacheDir = createTempDir(); try ( SearchableSnapshotDirectory directory = new SearchableSnapshotDirectory( @@ -98,6 +108,7 @@ public void testRandomReads() throws IOException { () -> 0L, cacheService, cacheDir, + shardPath, threadPool ) ) { @@ -158,6 +169,13 @@ public void testThrowsEOFException() throws IOException { final BlobContainer blobContainer = singleBlobContainer(blobName, input); final ThreadPool threadPool = new TestThreadPool(getTestName(), SearchableSnapshots.executorBuilders()); + final Path shardDir; + try { + shardDir = new NodeEnvironment.NodePath(createTempDir()).resolve(shardId); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId); final Path cacheDir = createTempDir(); try ( SearchableSnapshotDirectory searchableSnapshotDirectory = new SearchableSnapshotDirectory( @@ -170,6 +188,7 @@ public void testThrowsEOFException() throws IOException { () -> 0L, cacheService, cacheDir, + shardPath, threadPool ) ) { diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 02d3a2e764d73..75c80c2d828f3 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.stats.IndexStats; @@ -28,9 +29,12 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.repositories.RepositoriesService; @@ -44,6 +48,8 @@ import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsResponse; import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -59,6 +65,7 @@ import java.util.concurrent.CyclicBarrier; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; import java.util.stream.StreamSupport; import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; @@ -68,6 +75,7 @@ import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -125,8 +133,16 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); assertThat(snapshotInfo.successfulShards(), greaterThan(0)); assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + ensureGreen(indexName); - assertAcked(client().admin().indices().prepareDelete(indexName)); + assertShardFolders(indexName, false); + + final boolean deletedBeforeMount = randomBoolean(); + if (deletedBeforeMount) { + assertAcked(client().admin().indices().prepareDelete(indexName)); + } else { + assertAcked(client().admin().indices().prepareClose(indexName)); + } final boolean cacheEnabled = randomBoolean(); logger.info("--> restoring index [{}] with cache [{}]", restoredIndexName, cacheEnabled ? "enabled" : "disabled"); @@ -187,9 +203,22 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { assertRecovered(restoredIndexName, originalAllHits, originalBarHits); assertSearchableSnapshotStats(restoredIndexName, cacheEnabled, nonCachedExtensions); - - assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(0)); - assertAcked(client().admin().indices().prepareAliases().addAlias(restoredIndexName, aliasName)); + ensureGreen(restoredIndexName); + assertShardFolders(restoredIndexName, true); + + if (deletedBeforeMount) { + assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(0)); + assertAcked(client().admin().indices().prepareAliases().addAlias(restoredIndexName, aliasName)); + } else if (indexName.equals(restoredIndexName) == false) { + assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(1)); + assertAcked( + client().admin() + .indices() + .prepareAliases() + .addAliasAction(IndicesAliasesRequest.AliasActions.remove().index(indexName).alias(aliasName).mustExist(true)) + .addAlias(restoredIndexName, aliasName) + ); + } assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(1)); assertRecovered(aliasName, originalAllHits, originalBarHits, false); @@ -275,6 +304,32 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { } + private void assertShardFolders(String indexName, boolean snapshotDirectory) throws IOException { + final Index restoredIndex = resolveIndex(indexName); + final String customDataPath = resolveCustomDataPath(indexName); + final ShardId shardId = new ShardId(restoredIndex, 0); + boolean shardFolderFound = false; + for (String node : internalCluster().getNodeNames()) { + final NodeEnvironment service = internalCluster().getInstance(NodeEnvironment.class, node); + final ShardPath shardPath = ShardPath.loadShardPath(logger, service, shardId, customDataPath); + if (shardPath != null && Files.exists(shardPath.getDataPath())) { + shardFolderFound = true; + assertEquals(snapshotDirectory, Files.notExists(shardPath.resolveIndex())); + + assertTrue(Files.exists(shardPath.resolveTranslog())); + try (Stream dir = Files.list(shardPath.resolveTranslog())) { + final long translogFiles = dir.filter(path -> path.getFileName().toString().contains("translog")).count(); + if (snapshotDirectory) { + assertEquals(2L, translogFiles); + } else { + assertThat(translogFiles, greaterThanOrEqualTo(2L)); + } + } + } + } + assertTrue("no shard folder found", shardFolderFound); + } + public void testCanMountSnapshotTakenWhileConcurrentlyIndexing() throws Exception { final String fsRepoName = randomAlphaOfLength(10); final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);