Skip to content

Commit 9e24a54

Browse files
ywelschywelsch
committed
Clean existing index folder when loading searchable snapshot (#60122)
Closing a regular index and mounting a snapshot-backed index into that existing index does not clean the existing index folders of those preexisting shards. This PR removes the existing Lucene / translog files once the searchable snapshot shard is starting up. Future PRs will make reuse of the existing index files to populate the cache.
1 parent d2ddf8c commit 9e24a54

File tree

6 files changed

+131
-5
lines changed

6 files changed

+131
-5
lines changed

‎test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2375,6 +2375,12 @@ public static Index resolveIndex(String index) {
23752375
return new Index(index, uuid);
23762376
}
23772377

2378+
public static String resolveCustomDataPath(String index) {
2379+
GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(index).get();
2380+
assertTrue("index " + index + " not found", getIndexResponse.getSettings().containsKey(index));
2381+
return getIndexResponse.getSettings().get(index).get(IndexMetadata.SETTING_DATA_PATH);
2382+
}
2383+
23782384
public static boolean inFipsJvm() {
23792385
return Boolean.parseBoolean(System.getProperty(FIPS_SYSPROP));
23802386
}

‎x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
import java.io.FileNotFoundException;
5454
import java.io.IOException;
5555
import java.io.InputStream;
56+
import java.io.UncheckedIOException;
5657
import java.nio.file.DirectoryStream;
5758
import java.nio.file.Files;
5859
import java.nio.file.Path;
@@ -113,6 +114,7 @@ public class SearchableSnapshotDirectory extends BaseDirectory {
113114
private final Set<String> excludedFileTypes;
114115
private final long uncachedChunkSize; // if negative use BlobContainer#readBlobPreferredLength, see #getUncachedChunkSize()
115116
private final Path cacheDir;
117+
private final ShardPath shardPath;
116118
private final AtomicBoolean closed;
117119

118120
// volatile fields are updated once under `this` lock, all together, iff loaded is not true.
@@ -130,6 +132,7 @@ public SearchableSnapshotDirectory(
130132
LongSupplier currentTimeNanosSupplier,
131133
CacheService cacheService,
132134
Path cacheDir,
135+
ShardPath shardPath,
133136
ThreadPool threadPool
134137
) {
135138
super(new SingleInstanceLockFactory());
@@ -142,6 +145,7 @@ public SearchableSnapshotDirectory(
142145
this.statsCurrentTimeNanosSupplier = Objects.requireNonNull(currentTimeNanosSupplier);
143146
this.cacheService = Objects.requireNonNull(cacheService);
144147
this.cacheDir = Objects.requireNonNull(cacheDir);
148+
this.shardPath = Objects.requireNonNull(shardPath);
145149
this.closed = new AtomicBoolean(false);
146150
this.useCache = SNAPSHOT_CACHE_ENABLED_SETTING.get(indexSettings);
147151
this.prewarmCache = useCache ? SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING.get(indexSettings) : false;
@@ -182,6 +186,7 @@ public boolean loadSnapshot() {
182186
this.blobContainer = blobContainerSupplier.get();
183187
this.snapshot = snapshotSupplier.get();
184188
this.loaded = true;
189+
cleanExistingRegularShardFiles();
185190
prewarmCache();
186191
}
187192
}
@@ -374,6 +379,14 @@ public String toString() {
374379
return this.getClass().getSimpleName() + "@snapshotId=" + snapshotId + " lockFactory=" + lockFactory;
375380
}
376381

382+
private void cleanExistingRegularShardFiles() {
383+
try {
384+
IOUtils.rm(shardPath.resolveIndex(), shardPath.resolveTranslog());
385+
} catch (IOException e) {
386+
throw new UncheckedIOException(e);
387+
}
388+
}
389+
377390
private void prewarmCache() {
378391
if (prewarmCache) {
379392
final BlockingQueue<Tuple<ActionListener<Void>, CheckedRunnable<Exception>>> queue = new LinkedBlockingQueue<>();
@@ -513,6 +526,7 @@ public static Directory create(
513526
currentTimeNanosSupplier,
514527
cache,
515528
cacheDir,
529+
shardPath,
516530
threadPool
517531
)
518532
);

‎x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@
1616
import org.elasticsearch.common.unit.ByteSizeUnit;
1717
import org.elasticsearch.common.unit.ByteSizeValue;
1818
import org.elasticsearch.common.unit.TimeValue;
19+
import org.elasticsearch.env.NodeEnvironment;
1920
import org.elasticsearch.index.shard.ShardId;
21+
import org.elasticsearch.index.shard.ShardPath;
2022
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
2123
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
2224
import org.elasticsearch.index.store.cache.TestUtils;
@@ -28,7 +30,9 @@
2830
import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService;
2931

3032
import java.io.IOException;
33+
import java.io.UncheckedIOException;
3134
import java.nio.charset.StandardCharsets;
35+
import java.nio.file.Path;
3236
import java.util.Collections;
3337
import java.util.List;
3438
import java.util.concurrent.atomic.AtomicLong;
@@ -598,6 +602,14 @@ private static void executeTestCase(
598602
final StoreFileMetadata metadata = new StoreFileMetadata(fileName, fileContent.length, "_checksum", Version.CURRENT.luceneVersion);
599603
final List<FileInfo> files = Collections.singletonList(new FileInfo(blobName, metadata, new ByteSizeValue(fileContent.length)));
600604
final BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getName(), 0L, files, 0L, 0L, 0, 0L);
605+
final Path shardDir;
606+
try {
607+
shardDir = new NodeEnvironment.NodePath(createTempDir()).resolve(shardId);
608+
} catch (IOException e) {
609+
throw new UncheckedIOException(e);
610+
}
611+
final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId);
612+
final Path cacheDir = createTempDir();
601613

602614
try (
603615
CacheService ignored = cacheService;
@@ -610,7 +622,8 @@ private static void executeTestCase(
610622
indexSettings,
611623
statsCurrentTimeNanos,
612624
cacheService,
613-
createTempDir(),
625+
cacheDir,
626+
shardPath,
614627
threadPool
615628
) {
616629
@Override

‎x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,10 +58,12 @@
5858
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
5959
import org.elasticsearch.core.internal.io.IOUtils;
6060
import org.elasticsearch.env.Environment;
61+
import org.elasticsearch.env.NodeEnvironment;
6162
import org.elasticsearch.index.Index;
6263
import org.elasticsearch.index.IndexSettings;
6364
import org.elasticsearch.index.seqno.SequenceNumbers;
6465
import org.elasticsearch.index.shard.ShardId;
66+
import org.elasticsearch.index.shard.ShardPath;
6567
import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
6668
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
6769
import org.elasticsearch.index.store.cache.TestUtils;
@@ -86,6 +88,7 @@
8688
import java.io.EOFException;
8789
import java.io.FileNotFoundException;
8890
import java.io.IOException;
91+
import java.io.UncheckedIOException;
8992
import java.nio.charset.StandardCharsets;
9093
import java.nio.file.DirectoryStream;
9194
import java.nio.file.Files;
@@ -549,6 +552,13 @@ protected void assertSnapshotOrGenericThread() {
549552
final BlobContainer blobContainer = repository.shardContainer(indexId, shardId.id());
550553
final BlobStoreIndexShardSnapshot snapshot = repository.loadShardSnapshot(blobContainer, snapshotId);
551554

555+
final Path shardDir;
556+
try {
557+
shardDir = new NodeEnvironment.NodePath(createTempDir()).resolve(shardId);
558+
} catch (IOException e) {
559+
throw new UncheckedIOException(e);
560+
}
561+
final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId);
552562
final Path cacheDir = createTempDir();
553563
final CacheService cacheService = TestUtils.createDefaultCacheService();
554564
releasables.add(cacheService);
@@ -568,6 +578,7 @@ protected void assertSnapshotOrGenericThread() {
568578
() -> 0L,
569579
cacheService,
570580
cacheDir,
581+
shardPath,
571582
threadPool
572583
)
573584
) {
@@ -638,6 +649,13 @@ public void testClearCache() throws Exception {
638649
final IndexId indexId = new IndexId("_id", "_uuid");
639650
final ShardId shardId = new ShardId(new Index("_name", "_id"), 0);
640651

652+
final Path shardDir;
653+
try {
654+
shardDir = new NodeEnvironment.NodePath(createTempDir()).resolve(shardId);
655+
} catch (IOException e) {
656+
throw new UncheckedIOException(e);
657+
}
658+
final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId);
641659
final Path cacheDir = createTempDir();
642660
final ThreadPool threadPool = new TestThreadPool(getTestName(), SearchableSnapshots.executorBuilders());
643661
try (
@@ -656,6 +674,7 @@ public void testClearCache() throws Exception {
656674
() -> 0L,
657675
cacheService,
658676
cacheDir,
677+
shardPath,
659678
threadPool
660679
)
661680
) {

‎x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CachedBlobContainerIndexInputTests.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
import org.elasticsearch.common.lucene.store.ESIndexInputTestCase;
1313
import org.elasticsearch.common.settings.Settings;
1414
import org.elasticsearch.common.unit.ByteSizeValue;
15+
import org.elasticsearch.env.NodeEnvironment;
1516
import org.elasticsearch.index.shard.ShardId;
17+
import org.elasticsearch.index.shard.ShardPath;
1618
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
1719
import org.elasticsearch.index.store.SearchableSnapshotDirectory;
1820
import org.elasticsearch.index.store.StoreFileMetadata;
@@ -27,6 +29,7 @@
2729
import java.io.FilterInputStream;
2830
import java.io.IOException;
2931
import java.io.InputStream;
32+
import java.io.UncheckedIOException;
3033
import java.nio.charset.StandardCharsets;
3134
import java.nio.file.Path;
3235
import java.util.HashSet;
@@ -83,6 +86,13 @@ public void testRandomReads() throws IOException {
8386
blobContainer = singleBlobContainer;
8487
}
8588

89+
final Path shardDir;
90+
try {
91+
shardDir = new NodeEnvironment.NodePath(createTempDir()).resolve(shardId);
92+
} catch (IOException e) {
93+
throw new UncheckedIOException(e);
94+
}
95+
final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId);
8696
final Path cacheDir = createTempDir();
8797
try (
8898
SearchableSnapshotDirectory directory = new SearchableSnapshotDirectory(
@@ -98,6 +108,7 @@ public void testRandomReads() throws IOException {
98108
() -> 0L,
99109
cacheService,
100110
cacheDir,
111+
shardPath,
101112
threadPool
102113
)
103114
) {
@@ -158,6 +169,13 @@ public void testThrowsEOFException() throws IOException {
158169

159170
final BlobContainer blobContainer = singleBlobContainer(blobName, input);
160171
final ThreadPool threadPool = new TestThreadPool(getTestName(), SearchableSnapshots.executorBuilders());
172+
final Path shardDir;
173+
try {
174+
shardDir = new NodeEnvironment.NodePath(createTempDir()).resolve(shardId);
175+
} catch (IOException e) {
176+
throw new UncheckedIOException(e);
177+
}
178+
final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId);
161179
final Path cacheDir = createTempDir();
162180
try (
163181
SearchableSnapshotDirectory searchableSnapshotDirectory = new SearchableSnapshotDirectory(
@@ -170,6 +188,7 @@ public void testThrowsEOFException() throws IOException {
170188
() -> 0L,
171189
cacheService,
172190
cacheDir,
191+
shardPath,
173192
threadPool
174193
)
175194
) {

‎x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java‎

Lines changed: 59 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.ResourceNotFoundException;
1212
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
1313
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
14+
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
1415
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
1516
import org.elasticsearch.action.admin.indices.shrink.ResizeType;
1617
import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -25,9 +26,12 @@
2526
import org.elasticsearch.common.unit.ByteSizeValue;
2627
import org.elasticsearch.common.util.concurrent.AtomicArray;
2728
import org.elasticsearch.env.Environment;
29+
import org.elasticsearch.env.NodeEnvironment;
2830
import org.elasticsearch.index.Index;
2931
import org.elasticsearch.index.IndexModule;
3032
import org.elasticsearch.index.IndexSettings;
33+
import org.elasticsearch.index.shard.ShardId;
34+
import org.elasticsearch.index.shard.ShardPath;
3135
import org.elasticsearch.indices.IndicesService;
3236
import org.elasticsearch.indices.recovery.RecoveryState;
3337
import org.elasticsearch.repositories.RepositoriesService;
@@ -41,6 +45,8 @@
4145
import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsResponse;
4246
import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService;
4347

48+
import java.io.IOException;
49+
import java.nio.file.Files;
4450
import java.nio.file.Path;
4551
import java.util.ArrayList;
4652
import java.util.Arrays;
@@ -54,6 +60,7 @@
5460
import java.util.concurrent.CyclicBarrier;
5561
import java.util.stream.Collectors;
5662
import java.util.stream.IntStream;
63+
import java.util.stream.Stream;
5764
import java.util.stream.StreamSupport;
5865

5966
import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING;
@@ -63,6 +70,7 @@
6370
import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY;
6471
import static org.hamcrest.Matchers.equalTo;
6572
import static org.hamcrest.Matchers.greaterThan;
73+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
6674
import static org.hamcrest.Matchers.hasSize;
6775
import static org.hamcrest.Matchers.lessThanOrEqualTo;
6876

@@ -120,8 +128,16 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception {
120128
final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
121129
assertThat(snapshotInfo.successfulShards(), greaterThan(0));
122130
assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
131+
ensureGreen(indexName);
123132

124-
assertAcked(client().admin().indices().prepareDelete(indexName));
133+
assertShardFolders(indexName, false);
134+
135+
final boolean deletedBeforeMount = randomBoolean();
136+
if (deletedBeforeMount) {
137+
assertAcked(client().admin().indices().prepareDelete(indexName));
138+
} else {
139+
assertAcked(client().admin().indices().prepareClose(indexName));
140+
}
125141

126142
final boolean cacheEnabled = randomBoolean();
127143
logger.info("--> restoring index [{}] with cache [{}]", restoredIndexName, cacheEnabled ? "enabled" : "disabled");
@@ -182,9 +198,22 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception {
182198

183199
assertRecovered(restoredIndexName, originalAllHits, originalBarHits);
184200
assertSearchableSnapshotStats(restoredIndexName, cacheEnabled, nonCachedExtensions);
185-
186-
assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(0));
187-
assertAcked(client().admin().indices().prepareAliases().addAlias(restoredIndexName, aliasName));
201+
ensureGreen(restoredIndexName);
202+
assertShardFolders(restoredIndexName, true);
203+
204+
if (deletedBeforeMount) {
205+
assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(0));
206+
assertAcked(client().admin().indices().prepareAliases().addAlias(restoredIndexName, aliasName));
207+
} else if (indexName.equals(restoredIndexName) == false) {
208+
assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(1));
209+
assertAcked(
210+
client().admin()
211+
.indices()
212+
.prepareAliases()
213+
.addAliasAction(IndicesAliasesRequest.AliasActions.remove().index(indexName).alias(aliasName).mustExist(true))
214+
.addAlias(restoredIndexName, aliasName)
215+
);
216+
}
188217
assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(1));
189218
assertRecovered(aliasName, originalAllHits, originalBarHits, false);
190219

@@ -270,6 +299,32 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception {
270299

271300
}
272301

302+
private void assertShardFolders(String indexName, boolean snapshotDirectory) throws IOException {
303+
final Index restoredIndex = resolveIndex(indexName);
304+
final String customDataPath = resolveCustomDataPath(indexName);
305+
final ShardId shardId = new ShardId(restoredIndex, 0);
306+
boolean shardFolderFound = false;
307+
for (String node : internalCluster().getNodeNames()) {
308+
final NodeEnvironment service = internalCluster().getInstance(NodeEnvironment.class, node);
309+
final ShardPath shardPath = ShardPath.loadShardPath(logger, service, shardId, customDataPath);
310+
if (shardPath != null && Files.exists(shardPath.getDataPath())) {
311+
shardFolderFound = true;
312+
assertEquals(snapshotDirectory, Files.notExists(shardPath.resolveIndex()));
313+
314+
assertTrue(Files.exists(shardPath.resolveTranslog()));
315+
try (Stream<Path> dir = Files.list(shardPath.resolveTranslog())) {
316+
final long translogFiles = dir.filter(path -> path.getFileName().toString().contains("translog")).count();
317+
if (snapshotDirectory) {
318+
assertEquals(2L, translogFiles);
319+
} else {
320+
assertThat(translogFiles, greaterThanOrEqualTo(2L));
321+
}
322+
}
323+
}
324+
}
325+
assertTrue("no shard folder found", shardFolderFound);
326+
}
327+
273328
public void testCanMountSnapshotTakenWhileConcurrentlyIndexing() throws Exception {
274329
final String fsRepoName = randomAlphaOfLength(10);
275330
final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);

0 commit comments

Comments
 (0)