Skip to content

Commit 78a590f

Browse files
original-brownbearoriginal-brownbear
authored
Add Restore Operation to SnapshotResiliencyTests (#40634)
* Add Restore Operation to SnapshotResiliencyTests * Expand the successful snapshot test case to also include restoring the snapshop * Add indexing of documents as well to be able to meaningfully verify the restore * This is part of the larger effort to test eventually consistent blob stores in #39504
1 parent efe47ce commit 78a590f

File tree

1 file changed

+149
-15
lines changed

1 file changed

+149
-15
lines changed

‎server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java‎

Lines changed: 149 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -34,18 +34,42 @@
3434
import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotAction;
3535
import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
3636
import org.elasticsearch.action.admin.cluster.snapshots.delete.TransportDeleteSnapshotAction;
37+
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotAction;
38+
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
39+
import org.elasticsearch.action.admin.cluster.snapshots.restore.TransportRestoreSnapshotAction;
3740
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
3841
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
3942
import org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction;
4043
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
4144
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
4245
import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
46+
import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
47+
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
48+
import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
49+
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction;
50+
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
51+
import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction;
4352
import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction;
4453
import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction;
54+
import org.elasticsearch.action.bulk.BulkAction;
55+
import org.elasticsearch.action.bulk.BulkRequest;
56+
import org.elasticsearch.action.bulk.TransportBulkAction;
57+
import org.elasticsearch.action.bulk.TransportShardBulkAction;
58+
import org.elasticsearch.action.index.IndexRequest;
4559
import org.elasticsearch.action.resync.TransportResyncReplicationAction;
60+
import org.elasticsearch.action.search.SearchAction;
61+
import org.elasticsearch.action.search.SearchExecutionStatsCollector;
62+
import org.elasticsearch.action.search.SearchPhaseController;
63+
import org.elasticsearch.action.search.SearchRequest;
64+
import org.elasticsearch.action.search.SearchTransportService;
65+
import org.elasticsearch.action.search.TransportSearchAction;
4666
import org.elasticsearch.action.support.ActionFilters;
4767
import org.elasticsearch.action.support.ActiveShardCount;
68+
import org.elasticsearch.action.support.AutoCreateIndex;
69+
import org.elasticsearch.action.support.DestructiveOperations;
4870
import org.elasticsearch.action.support.TransportAction;
71+
import org.elasticsearch.action.support.WriteRequest;
72+
import org.elasticsearch.action.update.UpdateHelper;
4973
import org.elasticsearch.client.AdminClient;
5074
import org.elasticsearch.client.node.NodeClient;
5175
import org.elasticsearch.cluster.ClusterModule;
@@ -54,6 +78,7 @@
5478
import org.elasticsearch.cluster.ESAllocationTestCase;
5579
import org.elasticsearch.cluster.NodeConnectionsService;
5680
import org.elasticsearch.cluster.SnapshotsInProgress;
81+
import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
5782
import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
5883
import org.elasticsearch.cluster.action.shard.ShardStateAction;
5984
import org.elasticsearch.cluster.coordination.ClusterBootstrapService;
@@ -68,6 +93,8 @@
6893
import org.elasticsearch.cluster.metadata.IndexMetaData;
6994
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
7095
import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
96+
import org.elasticsearch.cluster.metadata.MetaDataDeleteIndexService;
97+
import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
7198
import org.elasticsearch.cluster.metadata.MetaDataMappingService;
7299
import org.elasticsearch.cluster.node.DiscoveryNode;
73100
import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -100,7 +127,9 @@
100127
import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction;
101128
import org.elasticsearch.index.seqno.RetentionLeaseSyncAction;
102129
import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
130+
import org.elasticsearch.indices.IndicesModule;
103131
import org.elasticsearch.indices.IndicesService;
132+
import org.elasticsearch.indices.analysis.AnalysisModule;
104133
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
105134
import org.elasticsearch.indices.cluster.FakeThreadPoolMasterService;
106135
import org.elasticsearch.indices.cluster.IndicesClusterStateService;
@@ -109,13 +138,16 @@
109138
import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
110139
import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
111140
import org.elasticsearch.indices.recovery.RecoverySettings;
112-
import org.elasticsearch.plugins.MapperPlugin;
141+
import org.elasticsearch.ingest.IngestService;
142+
import org.elasticsearch.node.ResponseCollectorService;
113143
import org.elasticsearch.plugins.PluginsService;
114144
import org.elasticsearch.repositories.RepositoriesService;
115145
import org.elasticsearch.repositories.Repository;
116146
import org.elasticsearch.repositories.fs.FsRepository;
117147
import org.elasticsearch.script.ScriptService;
118148
import org.elasticsearch.search.SearchService;
149+
import org.elasticsearch.search.builder.SearchSourceBuilder;
150+
import org.elasticsearch.search.fetch.FetchPhase;
119151
import org.elasticsearch.test.ESTestCase;
120152
import org.elasticsearch.test.disruption.DisruptableMockTransport;
121153
import org.elasticsearch.test.disruption.NetworkDisruption;
@@ -138,10 +170,12 @@
138170
import java.util.LinkedHashMap;
139171
import java.util.List;
140172
import java.util.Map;
173+
import java.util.Objects;
141174
import java.util.Optional;
142175
import java.util.Set;
143176
import java.util.concurrent.TimeUnit;
144177
import java.util.concurrent.atomic.AtomicBoolean;
178+
import java.util.concurrent.atomic.AtomicInteger;
145179
import java.util.function.Consumer;
146180
import java.util.function.Supplier;
147181
import java.util.stream.Collectors;
@@ -177,18 +211,20 @@ public void stopServices() {
177211
testClusterNodes.nodes.values().forEach(TestClusterNode::stop);
178212
}
179213

180-
public void testSuccessfulSnapshot() {
214+
public void testSuccessfulSnapshotAndRestore() {
181215
setupTestCluster(randomFrom(1, 3, 5), randomIntBetween(2, 10));
182216

183217
String repoName = "repo";
184218
String snapshotName = "snapshot";
185219
final String index = "test";
186220

187221
final int shards = randomIntBetween(1, 10);
188-
222+
final int documents = randomIntBetween(0, 100);
189223
TestClusterNode masterNode =
190224
testClusterNodes.currentMaster(testClusterNodes.nodes.values().iterator().next().clusterService.state());
191225
final AtomicBoolean createdSnapshot = new AtomicBoolean();
226+
final AtomicBoolean snapshotRestored = new AtomicBoolean();
227+
final AtomicBoolean documentCountVerified = new AtomicBoolean();
192228
masterNode.client.admin().cluster().preparePutRepository(repoName)
193229
.setType(FsRepository.TYPE).setSettings(Settings.builder().put("location", randomAlphaOfLength(10)))
194230
.execute(
@@ -197,12 +233,61 @@ public void testSuccessfulSnapshot() {
197233
new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL)
198234
.settings(defaultIndexSettings(shards)),
199235
assertNoFailureListener(
200-
() -> masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName)
201-
.execute(assertNoFailureListener(() -> createdSnapshot.set(true)))))));
202-
203-
deterministicTaskQueue.runAllRunnableTasks();
204-
236+
() -> {
237+
final Runnable afterIndexing = () ->
238+
masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName)
239+
.setWaitForCompletion(true).execute(assertNoFailureListener(() -> {
240+
createdSnapshot.set(true);
241+
masterNode.client.admin().indices().delete(
242+
new DeleteIndexRequest(index),
243+
assertNoFailureListener(() -> masterNode.client.admin().cluster().restoreSnapshot(
244+
new RestoreSnapshotRequest(repoName, snapshotName).waitForCompletion(true),
245+
assertNoFailureListener(restoreSnapshotResponse -> {
246+
snapshotRestored.set(true);
247+
assertEquals(shards, restoreSnapshotResponse.getRestoreInfo().totalShards());
248+
masterNode.client.search(
249+
new SearchRequest(index).source(
250+
new SearchSourceBuilder().size(0).trackTotalHits(true)
251+
),
252+
assertNoFailureListener(r -> {
253+
assertEquals(
254+
(long) documents,
255+
Objects.requireNonNull(r.getHits().getTotalHits()).value
256+
);
257+
documentCountVerified.set(true);
258+
}));
259+
})
260+
)));
261+
}));
262+
final AtomicInteger countdown = new AtomicInteger(documents);
263+
masterNode.client.admin().indices().putMapping(
264+
new PutMappingRequest(index).type("_doc").source("foo", "type=text"),
265+
assertNoFailureListener(r -> {
266+
for (int i = 0; i < documents; ++i) {
267+
masterNode.client.bulk(
268+
new BulkRequest().add(new IndexRequest(index).source(
269+
Collections.singletonMap("foo", "bar" + i)))
270+
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE),
271+
assertNoFailureListener(
272+
bulkResponse -> {
273+
assertFalse(
274+
"Failures in bulkresponse: " + bulkResponse.buildFailureMessage(),
275+
bulkResponse.hasFailures());
276+
if (countdown.decrementAndGet() == 0) {
277+
afterIndexing.run();
278+
}
279+
}));
280+
}
281+
if (documents == 0) {
282+
afterIndexing.run();
283+
}
284+
}
285+
));
286+
}))));
287+
runUntil(documentCountVerified::get, TimeUnit.MINUTES.toMillis(5L));
205288
assertTrue(createdSnapshot.get());
289+
assertTrue(snapshotRestored.get());
290+
assertTrue(documentCountVerified.get());
206291
SnapshotsInProgress finalSnapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE);
207292
assertFalse(finalSnapshotsInProgress.entries().stream().anyMatch(entry -> entry.state().completed() == false));
208293
final Repository repository = masterNode.repositoriesService.repository(repoName);
@@ -236,7 +321,6 @@ public void testSnapshotWithNodeDisconnects() {
236321
.execute(
237322
assertNoFailureListener(
238323
() -> masterNode.client.admin().indices().create(
239-
240324
new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL)
241325
.settings(defaultIndexSettings(shards)),
242326
assertNoFailureListener(
@@ -831,6 +915,8 @@ protected void assertSnapshotOrGenericThread() {
831915
allocationService = ESAllocationTestCase.createAllocationService(settings);
832916
final IndexScopedSettings indexScopedSettings =
833917
new IndexScopedSettings(settings, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
918+
final BigArrays bigArrays = new BigArrays(new PageCacheRecycler(settings), null, "test");
919+
final MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
834920
indicesService = new IndicesService(
835921
settings,
836922
mock(PluginsService.class),
@@ -839,12 +925,12 @@ protected void assertSnapshotOrGenericThread() {
839925
new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(),
840926
emptyMap(), emptyMap(), emptyMap(), emptyMap()),
841927
indexNameExpressionResolver,
842-
new MapperRegistry(emptyMap(), emptyMap(), MapperPlugin.NOOP_FIELD_FILTER),
928+
mapperRegistry,
843929
namedWriteableRegistry,
844930
threadPool,
845931
indexScopedSettings,
846932
new NoneCircuitBreakerService(),
847-
new BigArrays(new PageCacheRecycler(settings), null, "test"),
933+
bigArrays,
848934
scriptService,
849935
client,
850936
new MetaStateService(nodeEnv, namedXContentRegistry),
@@ -861,14 +947,15 @@ protected void assertSnapshotOrGenericThread() {
861947
new RoutingService(clusterService, allocationService),
862948
threadPool
863949
);
950+
final MetaDataMappingService metaDataMappingService = new MetaDataMappingService(clusterService, indicesService);
864951
indicesClusterStateService = new IndicesClusterStateService(
865952
settings,
866953
indicesService,
867954
clusterService,
868955
threadPool,
869956
new PeerRecoveryTargetService(threadPool, transportService, recoverySettings, clusterService),
870957
shardStateAction,
871-
new NodeMappingRefreshAction(transportService, new MetaDataMappingService(clusterService, indicesService)),
958+
new NodeMappingRefreshAction(transportService, metaDataMappingService),
872959
repositoriesService,
873960
mock(SearchService.class),
874961
new SyncedFlushService(indicesService, clusterService, transportService, indexNameExpressionResolver),
@@ -913,14 +1000,61 @@ protected void assertSnapshotOrGenericThread() {
9131000
actionFilters,
9141001
indexNameExpressionResolver));
9151002
Map<Action, TransportAction> actions = new HashMap<>();
1003+
final MetaDataCreateIndexService metaDataCreateIndexService = new MetaDataCreateIndexService(settings, clusterService,
1004+
indicesService,
1005+
allocationService, new AliasValidator(), environment, indexScopedSettings,
1006+
threadPool, namedXContentRegistry, false);
9161007
actions.put(CreateIndexAction.INSTANCE,
9171008
new TransportCreateIndexAction(
9181009
transportService, clusterService, threadPool,
919-
new MetaDataCreateIndexService(settings, clusterService, indicesService,
920-
allocationService, new AliasValidator(), environment, indexScopedSettings,
921-
threadPool, namedXContentRegistry, false),
1010+
metaDataCreateIndexService,
9221011
actionFilters, indexNameExpressionResolver
9231012
));
1013+
final MappingUpdatedAction mappingUpdatedAction = new MappingUpdatedAction(settings, clusterSettings);
1014+
mappingUpdatedAction.setClient(client);
1015+
final TransportShardBulkAction transportShardBulkAction = new TransportShardBulkAction(settings, transportService,
1016+
clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedAction, new UpdateHelper(scriptService),
1017+
actionFilters, indexNameExpressionResolver);
1018+
actions.put(BulkAction.INSTANCE,
1019+
new TransportBulkAction(threadPool, transportService, clusterService,
1020+
new IngestService(
1021+
clusterService, threadPool, environment, scriptService,
1022+
new AnalysisModule(environment, Collections.emptyList()).getAnalysisRegistry(),
1023+
Collections.emptyList()),
1024+
transportShardBulkAction, client, actionFilters, indexNameExpressionResolver,
1025+
new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver)
1026+
));
1027+
final RestoreService restoreService = new RestoreService(
1028+
clusterService, repositoriesService, allocationService,
1029+
metaDataCreateIndexService,
1030+
new MetaDataIndexUpgradeService(
1031+
settings, namedXContentRegistry,
1032+
mapperRegistry,
1033+
indexScopedSettings,
1034+
Collections.emptyList()
1035+
),
1036+
clusterSettings
1037+
);
1038+
actions.put(PutMappingAction.INSTANCE,
1039+
new TransportPutMappingAction(transportService, clusterService, threadPool, metaDataMappingService,
1040+
actionFilters, indexNameExpressionResolver, new TransportPutMappingAction.RequestValidators(Collections.emptyList())));
1041+
final ResponseCollectorService responseCollectorService = new ResponseCollectorService(clusterService);
1042+
final SearchTransportService searchTransportService = new SearchTransportService(transportService,
1043+
SearchExecutionStatsCollector.makeWrapper(responseCollectorService));
1044+
final SearchService searchService = new SearchService(clusterService, indicesService, threadPool, scriptService,
1045+
bigArrays, new FetchPhase(Collections.emptyList()), responseCollectorService);
1046+
actions.put(SearchAction.INSTANCE,
1047+
new TransportSearchAction(threadPool, transportService, searchService,
1048+
searchTransportService, new SearchPhaseController(searchService::createReduceContext), clusterService,
1049+
actionFilters, indexNameExpressionResolver));
1050+
actions.put(RestoreSnapshotAction.INSTANCE,
1051+
new TransportRestoreSnapshotAction(transportService, clusterService, threadPool, restoreService, actionFilters,
1052+
indexNameExpressionResolver));
1053+
actions.put(DeleteIndexAction.INSTANCE,
1054+
new TransportDeleteIndexAction(
1055+
transportService, clusterService, threadPool,
1056+
new MetaDataDeleteIndexService(settings, clusterService, allocationService), actionFilters,
1057+
indexNameExpressionResolver, new DestructiveOperations(settings, clusterSettings)));
9241058
actions.put(PutRepositoryAction.INSTANCE,
9251059
new TransportPutRepositoryAction(
9261060
transportService, clusterService, repositoriesService, threadPool,

0 commit comments

Comments
 (0)