From 80edff13c7b9caeeee6031338226247e0ab424bb Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 10 May 2019 18:43:52 +0200 Subject: [PATCH 01/35] Get snapshots support for multiple repositories --- .../client/SnapshotRequestConverters.java | 2 +- .../org/elasticsearch/client/SnapshotIT.java | 43 ++-- .../SnapshotRequestConvertersTests.java | 10 +- .../SnapshotClientDocumentationIT.java | 4 +- docs/reference/cat/snapshots.asciidoc | 6 +- .../url/URLSnapshotRestoreTests.java | 10 +- .../repositories/hdfs/HdfsTests.java | 2 +- .../test/cat.snapshots/10_basic.yml | 12 +- .../get/GetRepositoriesResponse.java | 4 + .../snapshots/get/GetSnapshotsRequest.java | 54 +++-- .../get/GetSnapshotsRequestBuilder.java | 21 +- .../snapshots/get/GetSnapshotsResponse.java | 229 ++++++++++++++---- .../get/TransportGetSnapshotsAction.java | 164 ++++++++----- .../client/ClusterAdminClient.java | 8 +- .../org/elasticsearch/client/Requests.java | 8 +- .../client/support/AbstractClient.java | 4 +- .../admin/cluster/RestGetSnapshotsAction.java | 4 +- .../rest/action/cat/RestSnapshotAction.java | 65 +++-- .../cluster/snapshots/SnapshotBlocksIT.java | 4 +- .../get/GetSnapshotsResponseTests.java | 88 ++++++- .../cluster/shards/ClusterShardLimitIT.java | 2 +- .../discovery/SnapshotDisruptionIT.java | 2 +- .../indices/recovery/IndexRecoveryIT.java | 2 +- .../AbstractSnapshotIntegTestCase.java | 2 +- .../DedicatedClusterSnapshotRestoreIT.java | 29 ++- ...etadataLoadingDuringSnapshotRestoreIT.java | 2 +- .../MinThreadsSnapshotRestoreIT.java | 6 +- .../SharedClusterSnapshotRestoreIT.java | 220 ++++++++++++----- .../snapshots/SnapshotShardsServiceIT.java | 4 +- .../test/rest/ESRestTestCase.java | 4 +- .../authz/SnapshotUserRoleIntegTests.java | 7 +- 31 files changed, 706 insertions(+), 316 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java index 93fb10bd56136..4b8974699b35d 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java @@ -104,7 +104,7 @@ static Request createSnapshot(CreateSnapshotRequest createSnapshotRequest) throw static Request getSnapshots(GetSnapshotsRequest getSnapshotsRequest) { RequestConverters.EndpointBuilder endpointBuilder = new RequestConverters.EndpointBuilder().addPathPartAsIs("_snapshot") - .addPathPart(getSnapshotsRequest.repository()); + .addCommaSeparatedPathParts(getSnapshotsRequest.repositories()); String endpoint; if (getSnapshotsRequest.snapshots().length == 0) { endpoint = endpointBuilder.addPathPart("_all").build(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java index 616850c513af7..3ecdd7ff9f488 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java @@ -41,14 +41,14 @@ import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.snapshots.RestoreInfo; +import org.mockito.internal.util.collections.Sets; import java.io.IOException; import java.util.Collections; -import java.util.stream.Collectors; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; public class SnapshotIT extends ESRestHighLevelClientTestCase { @@ -155,39 +155,48 @@ public void testCreateSnapshot() throws IOException { } public void testGetSnapshots() throws IOException { - String repository = "test_repository"; + String repository1 = "test_repository1"; + String repository2 = "test_repository2"; String snapshot1 = "test_snapshot1"; String snapshot2 = "test_snapshot2"; - AcknowledgedResponse putRepositoryResponse = createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}"); + AcknowledgedResponse putRepositoryResponse = + createTestRepository(repository1, FsRepository.TYPE, "{\"location\": \"loc1\"}"); assertTrue(putRepositoryResponse.isAcknowledged()); - CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository, snapshot1); + AcknowledgedResponse putRepositoryResponse2 = + createTestRepository(repository2, FsRepository.TYPE, "{\"location\": \"loc2\"}"); + assertTrue(putRepositoryResponse2.isAcknowledged()); + + CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository1, snapshot1); createSnapshotRequest1.waitForCompletion(true); CreateSnapshotResponse putSnapshotResponse1 = createTestSnapshot(createSnapshotRequest1); - CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository, snapshot2); + CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository2, snapshot2); createSnapshotRequest2.waitForCompletion(true); CreateSnapshotResponse putSnapshotResponse2 = createTestSnapshot(createSnapshotRequest2); // check that the request went ok without parsing JSON here. When using the high level client, check acknowledgement instead. assertEquals(RestStatus.OK, putSnapshotResponse1.status()); assertEquals(RestStatus.OK, putSnapshotResponse2.status()); - GetSnapshotsRequest request; - if (randomBoolean()) { - request = new GetSnapshotsRequest(repository); - } else if (randomBoolean()) { - request = new GetSnapshotsRequest(repository, new String[] {"_all"}); + GetSnapshotsRequest request = new GetSnapshotsRequest( + randomFrom(new String[]{"_all"}, new String[]{"*"}, new String[]{repository1, repository2}), + randomFrom(new String[]{"_all"}, new String[]{"*"}, new String[]{snapshot1, snapshot2}) + ); + request.ignoreUnavailable(true); - } else { - request = new GetSnapshotsRequest(repository, new String[] {snapshot1, snapshot2}); - } GetSnapshotsResponse response = execute(request, highLevelClient().snapshot()::get, highLevelClient().snapshot()::getAsync); - assertEquals(2, response.getSnapshots().size()); - assertThat(response.getSnapshots().stream().map((s) -> s.snapshotId().getName()).collect(Collectors.toList()), - contains("test_snapshot1", "test_snapshot2")); + assertThat(response.isFailed(), is(false)); + assertThat(response.getRepositories(), equalTo(Sets.newSet(repository1, repository2))); + + assertThat(response.getSnapshots(repository1), hasSize(1)); + assertThat(response.getSnapshots(repository1).get(0).snapshotId().getName(), equalTo(snapshot1)); + + assertThat(response.getSnapshots(repository2), hasSize(1)); + assertThat(response.getSnapshots(repository2).get(0).snapshotId().getName(), equalTo(snapshot2)); } + public void testSnapshotsStatus() throws IOException { String testRepository = "test"; String testSnapshot = "snapshot"; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java index 66720b70ee3a6..bea8835e093b8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java @@ -41,7 +41,6 @@ import java.io.IOException; import java.nio.file.Path; -import java.util.Arrays; import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -148,15 +147,16 @@ public void testCreateSnapshot() throws IOException { public void testGetSnapshots() { Map expectedParams = new HashMap<>(); - String repository = RequestConvertersTests.randomIndicesNames(1, 1)[0]; + String repository1 = randomAlphaOfLength(10); + String repository2 = randomAlphaOfLength(10); String snapshot1 = "snapshot1-" + randomAlphaOfLengthBetween(2, 5).toLowerCase(Locale.ROOT); String snapshot2 = "snapshot2-" + randomAlphaOfLengthBetween(2, 5).toLowerCase(Locale.ROOT); - String endpoint = String.format(Locale.ROOT, "/_snapshot/%s/%s,%s", repository, snapshot1, snapshot2); + String endpoint = String.format(Locale.ROOT, "/_snapshot/%s,%s/%s,%s", repository1, repository2, snapshot1, snapshot2); GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest(); - getSnapshotsRequest.repository(repository); - getSnapshotsRequest.snapshots(Arrays.asList(snapshot1, snapshot2).toArray(new String[0])); + getSnapshotsRequest.repositories(repository1, repository2); + getSnapshotsRequest.snapshots(new String[]{snapshot1, snapshot2}); RequestConvertersTests.setRandomMasterTimeout(getSnapshotsRequest, expectedParams); if (randomBoolean()) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java index d80c24be6618a..5a6016175bae2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java @@ -590,7 +590,7 @@ public void testSnapshotGetSnapshots() throws IOException { // end::get-snapshots-request // tag::get-snapshots-request-repositoryName - request.repository(repositoryName); // <1> + request.repositories(repositoryName); // <1> // end::get-snapshots-request-repositoryName // tag::get-snapshots-request-snapshots @@ -616,7 +616,7 @@ public void testSnapshotGetSnapshots() throws IOException { // end::get-snapshots-execute // tag::get-snapshots-response - List snapshotsInfos = response.getSnapshots(); + List snapshotsInfos = response.getSnapshots(repositoryName); SnapshotInfo snapshotInfo = snapshotsInfos.get(0); RestStatus restStatus = snapshotInfo.status(); // <1> SnapshotId snapshotId = snapshotInfo.snapshotId(); // <2> diff --git a/docs/reference/cat/snapshots.asciidoc b/docs/reference/cat/snapshots.asciidoc index 5677a0f2a7cd4..003514cc3a9d0 100644 --- a/docs/reference/cat/snapshots.asciidoc +++ b/docs/reference/cat/snapshots.asciidoc @@ -18,9 +18,9 @@ Which looks like: [source,txt] -------------------------------------------------- -id status start_epoch start_time end_epoch end_time duration indices successful_shards failed_shards total_shards -snap1 FAILED 1445616705 18:11:45 1445616978 18:16:18 4.6m 1 4 1 5 -snap2 SUCCESS 1445634298 23:04:58 1445634672 23:11:12 6.2m 2 10 0 10 +id repository status start_epoch start_time end_epoch end_time duration indices successful_shards failed_shards total_shards +snap1 repo1 FAILED 1445616705 18:11:45 1445616978 18:16:18 4.6m 1 4 1 5 +snap2 repo1 SUCCESS 1445634298 23:04:58 1445634672 23:11:12 6.2m 2 10 0 10 -------------------------------------------------- // TESTRESPONSE[s/FAILED/SUCCESS/ s/14456\d+/\\d+/ s/\d+(\.\d+)?(m|s|ms)/\\d+(\\.\\d+)?(m|s|ms)/] // TESTRESPONSE[s/\d+:\d+:\d+/\\d+:\\d+:\\d+/] diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java index ab9268b081456..160333d5346ed 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java @@ -88,7 +88,7 @@ public void testUrlRepository() throws Exception { .prepareGetSnapshots("test-repo") .setSnapshots("test-snap") .get() - .getSnapshots() + .getSnapshots("test-repo") .get(0) .state(); assertThat(state, equalTo(SnapshotState.SUCCESS)); @@ -116,8 +116,8 @@ public void testUrlRepository() throws Exception { logger.info("--> list available shapshots"); GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get(); - assertThat(getSnapshotsResponse.getSnapshots(), notNullValue()); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1)); + assertThat(getSnapshotsResponse.getSnapshots("url-repo"), notNullValue()); + assertThat(getSnapshotsResponse.getSnapshots("url-repo").size(), equalTo(1)); logger.info("--> delete snapshot"); AcknowledgedResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get(); @@ -125,7 +125,7 @@ public void testUrlRepository() throws Exception { logger.info("--> list available shapshot again, no snapshots should be returned"); getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get(); - assertThat(getSnapshotsResponse.getSnapshots(), notNullValue()); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0)); + assertThat(getSnapshotsResponse.getSnapshots("url-repo"), notNullValue()); + assertThat(getSnapshotsResponse.getSnapshots("url-repo").size(), equalTo(0)); } } diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java index 88454188da588..8c211a6cabb43 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java @@ -91,7 +91,7 @@ public void testSimpleWorkflow() { .prepareGetSnapshots("test-repo") .setSnapshots("test-snap") .get() - .getSnapshots() + .getSnapshots("test-repo") .get(0) .state(), equalTo(SnapshotState.SUCCESS)); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml index 6e03ceb98c716..01af7c14454a1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml @@ -1,5 +1,9 @@ --- "Help": + - skip: + version: " - 7.9.99" + reason: Repository field added in 8.0 + - do: cat.snapshots: help: true @@ -7,6 +11,7 @@ - match: $body: | /^ id .+ \n + repository .+ \n status .+ \n start_epoch .+ \n start_time .+ \n @@ -21,6 +26,9 @@ $/ --- "Test cat snapshots output": + - skip: + version: " - 7.9.99" + reason: Repository field added in 8.0 - do: snapshot.create_repository: @@ -74,6 +82,6 @@ - match: $body: | - /^ snap1\s+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n - snap2\s+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n + /^ snap1\s+ test_cat_snapshots_1\+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n + snap2\s+ test_cat_snapshots_1\+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n $/ diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java index 24228aa565871..add8e13b26ac6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java @@ -49,6 +49,10 @@ public class GetRepositoriesResponse extends ActionResponse implements ToXConten this.repositories = repositories; } + public GetRepositoriesResponse(StreamInput in) throws IOException { + readFrom(in); + } + /** * List of repositories to return * diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java index 41ae57031d320..7c3f73ecf08bf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.cluster.snapshots.get; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.common.Strings; @@ -37,8 +38,9 @@ public class GetSnapshotsRequest extends MasterNodeRequest public static final String ALL_SNAPSHOTS = "_all"; public static final String CURRENT_SNAPSHOT = "_current"; public static final boolean DEFAULT_VERBOSE_MODE = true; + public static final Version MULTIPLE_REPOSITORIES_SUPPORT_ADDED = Version.V_8_0_0; - private String repository; + private String[] repositories; private String[] snapshots = Strings.EMPTY_ARRAY; @@ -50,28 +52,32 @@ public GetSnapshotsRequest() { } /** - * Constructs a new get snapshots request with given repository name and list of snapshots + * Constructs a new get snapshots request with given repository names and list of snapshots * - * @param repository repository name + * @param repositories repository names * @param snapshots list of snapshots */ - public GetSnapshotsRequest(String repository, String[] snapshots) { - this.repository = repository; + public GetSnapshotsRequest(String[] repositories, String[] snapshots) { + this.repositories = repositories; this.snapshots = snapshots; } /** - * Constructs a new get snapshots request with given repository name + * Constructs a new get snapshots request with given repository names * - * @param repository repository name + * @param repositories repository names */ - public GetSnapshotsRequest(String repository) { - this.repository = repository; + public GetSnapshotsRequest(String... repositories) { + this.repositories = repositories; } public GetSnapshotsRequest(StreamInput in) throws IOException { super(in); - repository = in.readString(); + if (in.getVersion().onOrAfter(MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) { + repositories = in.readStringArray(); + } else { + repositories = new String[]{in.readString()}; + } snapshots = in.readStringArray(); ignoreUnavailable = in.readBoolean(); verbose = in.readBoolean(); @@ -80,7 +86,11 @@ public GetSnapshotsRequest(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeString(repository); + if (out.getVersion().onOrAfter(MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) { + out.writeStringArray(repositories); + } else { + out.writeString(repositories[0]); + } out.writeStringArray(snapshots); out.writeBoolean(ignoreUnavailable); out.writeBoolean(verbose); @@ -89,30 +99,30 @@ public void writeTo(StreamOutput out) throws IOException { @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; - if (repository == null) { - validationException = addValidationError("repository is missing", validationException); + if (repositories == null || repositories.length == 0) { + validationException = addValidationError("repositories are missing", validationException); } return validationException; } /** - * Sets repository name + * Sets repository names * - * @param repository repository name + * @param repositories repository names * @return this request */ - public GetSnapshotsRequest repository(String repository) { - this.repository = repository; + public GetSnapshotsRequest repositories(String... repositories) { + this.repositories = repositories; return this; } /** - * Returns repository name + * Returns repository names * - * @return repository name + * @return repository names */ - public String repository() { - return this.repository; + public String[] repositories() { + return this.repositories; } /** @@ -176,4 +186,4 @@ public boolean verbose() { public void readFrom(StreamInput in) throws IOException { throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); } -} +} \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java index 052f8da0c7508..e4219c858551a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java @@ -30,27 +30,20 @@ public class GetSnapshotsRequestBuilder extends MasterNodeOperationRequestBuilde GetSnapshotsResponse, GetSnapshotsRequestBuilder> { /** - * Constructs the new get snapshot request + * Constructs the new get snapshot request with specified repositories */ - public GetSnapshotsRequestBuilder(ElasticsearchClient client, GetSnapshotsAction action) { - super(client, action, new GetSnapshotsRequest()); + public GetSnapshotsRequestBuilder(ElasticsearchClient client, GetSnapshotsAction action, String... repositories) { + super(client, action, new GetSnapshotsRequest(repositories)); } /** - * Constructs the new get snapshot request with specified repository - */ - public GetSnapshotsRequestBuilder(ElasticsearchClient client, GetSnapshotsAction action, String repository) { - super(client, action, new GetSnapshotsRequest(repository)); - } - - /** - * Sets the repository name + * Sets the repository names * - * @param repository repository name + * @param repositories repository names * @return this builder */ - public GetSnapshotsRequestBuilder setRepository(String repository) { - request.repository(repository); + public GetSnapshotsRequestBuilder setRepositories(String... repositories) { + request.repositories(repositories); return this; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index 6f757cb60ca86..5c25e8dffc22b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -19,12 +19,13 @@ package org.elasticsearch.action.admin.cluster.snapshots.get; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -33,88 +34,220 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; -import java.util.Objects; +import java.util.Map; +import java.util.Set; /** * Get snapshots response */ public class GetSnapshotsResponse extends ActionResponse implements ToXContentObject { - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser GET_SNAPSHOT_PARSER = - new ConstructingObjectParser<>(GetSnapshotsResponse.class.getName(), true, - (args) -> new GetSnapshotsResponse((List) args[0])); + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(GetSnapshotsResponse.class.getName(), true, + (args) -> new GetSnapshotsResponse((List) args[0])); static { - GET_SNAPSHOT_PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), - (p, c) -> SnapshotInfo.SNAPSHOT_INFO_PARSER.apply(p, c).build(), new ParseField("snapshots")); + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), + (p, c) -> Response.fromXContent(p), new ParseField("responses")); } - private List snapshots = Collections.emptyList(); + public GetSnapshotsResponse(Map> successfulResponses, Map failedResponses) { + this.successfulResponses = successfulResponses; + this.failedResponses = failedResponses; + } + + private static class Response { + String repository; + List snapshots; + ElasticsearchException error; + + static final ConstructingObjectParser RESPONSE_PARSER = + new ConstructingObjectParser<>(Response.class.getName(), true, + (args) -> new Response((String) args[0], + (List) args[1], (ElasticsearchException) args[2])); + + static { + RESPONSE_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("repository")); + RESPONSE_PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> SnapshotInfo.SNAPSHOT_INFO_PARSER.apply(p, c).build(), new ParseField("snapshots")); + RESPONSE_PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> ElasticsearchException.fromXContent(p), new ParseField("error")); + } - GetSnapshotsResponse() { + Response(String repository, List snapshots, ElasticsearchException error) { + this.repository = repository; + this.snapshots = snapshots; + this.error = error; + } + + static Response fromXContent(XContentParser parser) throws IOException { + return RESPONSE_PARSER.parse(parser, null); + } } - GetSnapshotsResponse(List snapshots) { - this.snapshots = Collections.unmodifiableList(snapshots); + private Map> successfulResponses = Collections.emptyMap(); + private Map failedResponses = Collections.emptyMap(); + + private GetSnapshotsResponse(List responses) { + this.successfulResponses = new HashMap<>(); + this.failedResponses = new HashMap<>(); + for (Response response : responses) { + if (response.snapshots != null) { + this.successfulResponses.put(response.repository, response.snapshots); + } else { + this.failedResponses.put(response.repository, response.error); + } + } + } + + public GetSnapshotsResponse() { } /** - * Returns the list of snapshots - * - * @return the list of snapshots + * Returns list of snapshots for the specified repository. + * @param repo - repository name. + * @return list of snapshots. + * @throws IllegalArgumentException if there is no such repository in the response. + * @throws ElasticsearchException if an exception occurred when retrieving snapshots from the repository. */ - public List getSnapshots() { - return snapshots; + public List getSnapshots(String repo) { + List snapshots = successfulResponses.get(repo); + if (snapshots != null) { + return snapshots; + } + ElasticsearchException error = failedResponses.get(repo); + if (error == null) { + throw new IllegalArgumentException("No such repository"); + } + throw error; } - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - int size = in.readVInt(); - List builder = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - builder.add(new SnapshotInfo(in)); - } - snapshots = Collections.unmodifiableList(builder); + /** + * Returns list of repositories for both successful and unsuccessful responses. + */ + public Set getRepositories() { + return Sets.union(successfulResponses.keySet(), failedResponses.keySet()); } - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeVInt(snapshots.size()); - for (SnapshotInfo snapshotInfo : snapshots) { - snapshotInfo.writeTo(out); - } + /** + * Returns a map of repository name to the list of {@link SnapshotInfo} for each successful response. + */ + public Map> getSuccessfulResponses() { + return Map.copyOf(successfulResponses); + } + + /** + * Returns a map of repository name to {@link ElasticsearchException} for each unsuccessful response. + */ + public Map getFailedResponses() { + return Map.copyOf(failedResponses); + } + + /** + * Returns true if there is a least one failed response. + */ + public boolean isFailed() { + return failedResponses.isEmpty() == false; } @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.startArray("snapshots"); - for (SnapshotInfo snapshotInfo : snapshots) { - snapshotInfo.toXContent(builder, params); + builder.startArray("responses"); + + for (Map.Entry> snapshots : successfulResponses.entrySet()) { + builder.startObject(); + builder.field("repository", snapshots.getKey()); + builder.startArray("snapshots"); + for (SnapshotInfo snapshot : snapshots.getValue()) { + snapshot.toXContent(builder, params); + } + builder.endArray(); + builder.endObject(); + } + + for (Map.Entry error : failedResponses.entrySet()) { + builder.startObject(); + builder.field("repository", error.getKey()); + ElasticsearchException.generateFailureXContent(builder, params, error.getValue(), true); + builder.endObject(); } + builder.endArray(); builder.endObject(); return builder; } - public static GetSnapshotsResponse fromXContent(XContentParser parser) throws IOException { - return GET_SNAPSHOT_PARSER.parse(parser, null); + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + if (out.getVersion().onOrAfter(GetSnapshotsRequest.MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) { + out.writeVInt(successfulResponses.size()); + for (Map.Entry> snapshots : successfulResponses.entrySet()) { + out.writeString(snapshots.getKey()); + out.writeVInt(snapshots.getValue().size()); + for (SnapshotInfo snapshotInfo : snapshots.getValue()) { + snapshotInfo.writeTo(out); + } + } + out.writeVInt(failedResponses.size()); + for (Map.Entry error : failedResponses.entrySet()) { + out.writeString(error.getKey()); + out.writeException(error.getValue()); + } + } else { + if (successfulResponses.size() + failedResponses.size() != 1) { + throw new IllegalArgumentException("snapshots from more than one repository are requested"); + } + + if (successfulResponses.size() == 1) { + List snapshotInfos = successfulResponses.values().iterator().next(); + out.writeVInt(snapshotInfos.size()); + for (SnapshotInfo snapshotInfo : snapshotInfos) { + snapshotInfo.writeTo(out); + } + } + + throw failedResponses.values().iterator().next(); + } } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - GetSnapshotsResponse that = (GetSnapshotsResponse) o; - return Objects.equals(snapshots, that.snapshots); + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + if (in.getVersion().onOrAfter(GetSnapshotsRequest.MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) { + int successfulSize = in.readVInt(); + successfulResponses = new HashMap<>(successfulSize); + for (int i = 0; i < successfulSize; i++) { + String repository = in.readString(); + int size = in.readVInt(); + List snapshotInfos = new ArrayList<>(size); + for (int j = 0; j < size; j++) { + snapshotInfos.add(new SnapshotInfo(in)); + } + successfulResponses.put(repository, snapshotInfos); + } + + int failedSize = in.readVInt(); + failedResponses = new HashMap<>(failedSize); + for (int i = 0; i < failedSize; i++) { + String repository = in.readString(); + ElasticsearchException error = in.readException(); + failedResponses.put(repository, error); + } + } else { + int size = in.readVInt(); + List snapshots = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + snapshots.add(new SnapshotInfo(in)); + } + successfulResponses.put("unknown", snapshots); + } } - @Override - public int hashCode() { - return Objects.hash(snapshots); + public static GetSnapshotsResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); } -} +} \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 23ffbd0dd1e3c..1efd0497dd845 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -19,14 +19,21 @@ package org.elasticsearch.action.admin.cluster.snapshots.get; +import org.apache.logging.log4j.core.util.Throwables; import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; +import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesAction; +import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; +import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; @@ -46,6 +53,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import java.util.stream.Collectors; /** @@ -59,7 +68,7 @@ public TransportGetSnapshotsAction(TransportService transportService, ClusterSer ThreadPool threadPool, SnapshotsService snapshotsService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { super(GetSnapshotsAction.NAME, transportService, clusterService, threadPool, actionFilters, - GetSnapshotsRequest::new, indexNameExpressionResolver); + GetSnapshotsRequest::new, indexNameExpressionResolver); this.snapshotsService = snapshotsService; } @@ -81,73 +90,116 @@ protected ClusterBlockException checkBlock(GetSnapshotsRequest request, ClusterS @Override protected void masterOperation(final GetSnapshotsRequest request, final ClusterState state, final ActionListener listener) { - try { - final String repository = request.repository(); - final Map allSnapshotIds = new HashMap<>(); - final List currentSnapshots = new ArrayList<>(); - for (SnapshotInfo snapshotInfo : snapshotsService.currentSnapshots(repository)) { - SnapshotId snapshotId = snapshotInfo.snapshotId(); - allSnapshotIds.put(snapshotId.getName(), snapshotId); - currentSnapshots.add(snapshotInfo); - } + final String[] repositories = request.repositories(); + transportService.sendRequest(transportService.getLocalNode(), GetRepositoriesAction.NAME, + new GetRepositoriesRequest(repositories), + new ActionListenerResponseHandler<>( + ActionListener.wrap( + response -> + // switch to GENERIC thread pool because it might be long running operation + threadPool.executor(ThreadPool.Names.GENERIC).execute( + () -> getMultipleReposSnapshotInfo(response.repositories(), request.snapshots(), + request.ignoreUnavailable(), request.verbose(), listener)), + listener::onFailure), + GetRepositoriesResponse::new)); + } + + private void getMultipleReposSnapshotInfo(List repos, String[] snapshots, boolean ignoreUnavailable, + boolean verbose, ActionListener listener) { + List>> futures = new ArrayList<>(repos.size()); - final RepositoryData repositoryData; - if (isCurrentSnapshotsOnly(request.snapshots()) == false) { - repositoryData = snapshotsService.getRepositoryData(repository); - for (SnapshotId snapshotId : repositoryData.getAllSnapshotIds()) { - allSnapshotIds.put(snapshotId.getName(), snapshotId); + // run concurrently for all repos on SNAPSHOT thread pool + for (final RepositoryMetaData repo : repos) { + futures.add(threadPool.executor(ThreadPool.Names.SNAPSHOT).submit( + () -> getSingleRepoSnapshotInfo(repo.name(), snapshots, ignoreUnavailable, verbose))); + } + assert repos.size() == futures.size(); + + Map> successfulResponses = new HashMap<>(); + Map failedResponses = new HashMap<>(); + + for (int i = 0; i < repos.size(); i++) { + final String repo = repos.get(i).name(); + try { + successfulResponses.put(repo, futures.get(i).get()); + } catch (InterruptedException e) { + Throwables.rethrow(e); + } catch (ExecutionException e) { + if (e.getCause() instanceof ElasticsearchException) { + failedResponses.put(repo, (ElasticsearchException) e.getCause()); + } else { + Throwables.rethrow(e); } - } else { - repositoryData = null; } + } - final Set toResolve = new HashSet<>(); - if (isAllSnapshots(request.snapshots())) { - toResolve.addAll(allSnapshotIds.values()); - } else { - for (String snapshotOrPattern : request.snapshots()) { - if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) { - toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId).collect(Collectors.toList())); - } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) { - if (allSnapshotIds.containsKey(snapshotOrPattern)) { - toResolve.add(allSnapshotIds.get(snapshotOrPattern)); - } else if (request.ignoreUnavailable() == false) { - throw new SnapshotMissingException(repository, snapshotOrPattern); - } - } else { - for (Map.Entry entry : allSnapshotIds.entrySet()) { - if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) { - toResolve.add(entry.getValue()); - } + listener.onResponse(new GetSnapshotsResponse(successfulResponses, failedResponses)); + } + + private List getSingleRepoSnapshotInfo(String repo, String[] snapshots, boolean ignoreUnavailable, boolean verbose) { + final Map allSnapshotIds = new HashMap<>(); + final List currentSnapshots = new ArrayList<>(); + for (SnapshotInfo snapshotInfo : snapshotsService.currentSnapshots(repo)) { + SnapshotId snapshotId = snapshotInfo.snapshotId(); + allSnapshotIds.put(snapshotId.getName(), snapshotId); + currentSnapshots.add(snapshotInfo); + } + + final RepositoryData repositoryData; + if (isCurrentSnapshotsOnly(snapshots) == false) { + repositoryData = snapshotsService.getRepositoryData(repo); + for (SnapshotId snapshotId : repositoryData.getAllSnapshotIds()) { + allSnapshotIds.put(snapshotId.getName(), snapshotId); + } + } else { + repositoryData = null; + } + + final Set toResolve = new HashSet<>(); + if (isAllSnapshots(snapshots)) { + toResolve.addAll(allSnapshotIds.values()); + } else { + for (String snapshotOrPattern : snapshots) { + if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) { + toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId).collect(Collectors.toList())); + } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) { + if (allSnapshotIds.containsKey(snapshotOrPattern)) { + toResolve.add(allSnapshotIds.get(snapshotOrPattern)); + } else if (ignoreUnavailable == false) { + throw new SnapshotMissingException(repo, snapshotOrPattern); + } + } else { + for (Map.Entry entry : allSnapshotIds.entrySet()) { + if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) { + toResolve.add(entry.getValue()); } } } + } - if (toResolve.isEmpty() && request.ignoreUnavailable() == false && isCurrentSnapshotsOnly(request.snapshots()) == false) { - throw new SnapshotMissingException(repository, request.snapshots()[0]); - } + if (toResolve.isEmpty() && ignoreUnavailable == false && isCurrentSnapshotsOnly(snapshots) == false) { + throw new SnapshotMissingException(repo, snapshots[0]); } + } - final List snapshotInfos; - if (request.verbose()) { - final Set incompatibleSnapshots = repositoryData != null ? + final List snapshotInfos; + if (verbose) { + final Set incompatibleSnapshots = repositoryData != null ? new HashSet<>(repositoryData.getIncompatibleSnapshotIds()) : Collections.emptySet(); - snapshotInfos = snapshotsService.snapshots(repository, new ArrayList<>(toResolve), - incompatibleSnapshots, request.ignoreUnavailable()); + snapshotInfos = snapshotsService.snapshots(repo, new ArrayList<>(toResolve), + incompatibleSnapshots, ignoreUnavailable); + } else { + if (repositoryData != null) { + // want non-current snapshots as well, which are found in the repository data + snapshotInfos = buildSimpleSnapshotInfos(repo, toResolve, repositoryData, currentSnapshots); } else { - if (repositoryData != null) { - // want non-current snapshots as well, which are found in the repository data - snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots); - } else { - // only want current snapshots - snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList()); - CollectionUtil.timSort(snapshotInfos); - } + // only want current snapshots + snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList()); + CollectionUtil.timSort(snapshotInfos); } - listener.onResponse(new GetSnapshotsResponse(snapshotInfos)); - } catch (Exception e) { - listener.onFailure(e); } + + return snapshotInfos; } private boolean isAllSnapshots(String[] snapshots) { @@ -158,7 +210,7 @@ private boolean isCurrentSnapshotsOnly(String[] snapshots) { return (snapshots.length == 1 && GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshots[0])); } - private List buildSimpleSnapshotInfos(final Set toResolve, + private List buildSimpleSnapshotInfos(final String repository, final Set toResolve, final RepositoryData repositoryData, final List currentSnapshots) { List snapshotInfos = new ArrayList<>(); @@ -172,7 +224,7 @@ private List buildSimpleSnapshotInfos(final Set toReso for (SnapshotId snapshotId : repositoryData.getSnapshots(indexId)) { if (toResolve.contains(snapshotId)) { snapshotsToIndices.computeIfAbsent(snapshotId, (k) -> new ArrayList<>()) - .add(indexId.getName()); + .add(indexId.getName()); } } } diff --git a/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java index 42aaed10d6172..cd874b62a40b4 100644 --- a/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java @@ -484,19 +484,19 @@ public interface ClusterAdminClient extends ElasticsearchClient { CreateSnapshotRequestBuilder prepareCreateSnapshot(String repository, String name); /** - * Get snapshot. + * Get snapshots. */ ActionFuture getSnapshots(GetSnapshotsRequest request); /** - * Get snapshot. + * Get snapshots. */ void getSnapshots(GetSnapshotsRequest request, ActionListener listener); /** - * Get snapshot. + * Get snapshots. */ - GetSnapshotsRequestBuilder prepareGetSnapshots(String repository); + GetSnapshotsRequestBuilder prepareGetSnapshots(String... repository); /** * Delete snapshot. diff --git a/server/src/main/java/org/elasticsearch/client/Requests.java b/server/src/main/java/org/elasticsearch/client/Requests.java index 19ad2fb397edc..3e4ac2eafed59 100644 --- a/server/src/main/java/org/elasticsearch/client/Requests.java +++ b/server/src/main/java/org/elasticsearch/client/Requests.java @@ -494,13 +494,13 @@ public static CreateSnapshotRequest createSnapshotRequest(String repository, Str } /** - * Gets snapshots from repository + * Gets snapshots from repositories * - * @param repository repository name + * @param repositories repository names * @return get snapshot request */ - public static GetSnapshotsRequest getSnapshotsRequest(String repository) { - return new GetSnapshotsRequest(repository); + public static GetSnapshotsRequest getSnapshotsRequest(String... repositories) { + return new GetSnapshotsRequest(repositories); } /** diff --git a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java index e79f0567babe6..f72bae2731457 100644 --- a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -955,8 +955,8 @@ public void getSnapshots(GetSnapshotsRequest request, ActionListener - client.admin().cluster().getSnapshots(getSnapshotsRequest, new RestResponseListener(channel) { + client.admin().cluster().getSnapshots(getSnapshotsRequest, new RestResponseListener<>(channel) { @Override public RestResponse buildResponse(GetSnapshotsResponse getSnapshotsResponse) throws Exception { return RestTable.buildResponse(buildTable(request, getSnapshotsResponse), channel); @@ -84,6 +88,7 @@ protected Table getTableWithHeader(RestRequest request) { return new Table() .startHeaders() .addCell("id", "alias:id,snapshot;desc:unique snapshot") + .addCell("repository", "alias:re,repo;desc:repository name") .addCell("status", "alias:s,status;text-align:right;desc:snapshot name") .addCell("start_epoch", "alias:ste,startEpoch;desc:start time in seconds since 1970-01-01 00:00:00") .addCell("start_time", "alias:sti,startTime;desc:start time in HH:MM:SS") @@ -102,29 +107,41 @@ protected Table getTableWithHeader(RestRequest request) { private Table buildTable(RestRequest req, GetSnapshotsResponse getSnapshotsResponse) { Table table = getTableWithHeader(req); - for (SnapshotInfo snapshotStatus : getSnapshotsResponse.getSnapshots()) { - table.startRow(); - - table.addCell(snapshotStatus.snapshotId().getName()); - table.addCell(snapshotStatus.state()); - table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.startTime(), TimeUnit.MILLISECONDS)); - table.addCell(FORMATTER.format(Instant.ofEpochMilli(snapshotStatus.startTime()))); - table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.endTime(), TimeUnit.MILLISECONDS)); - table.addCell(FORMATTER.format(Instant.ofEpochMilli(snapshotStatus.endTime()))); - final long durationMillis; - if (snapshotStatus.state() == SnapshotState.IN_PROGRESS) { - durationMillis = System.currentTimeMillis() - snapshotStatus.startTime(); - } else { - durationMillis = snapshotStatus.endTime() - snapshotStatus.startTime(); + + if (getSnapshotsResponse.isFailed()) { + throw new ElasticsearchException( + "Repositories [" + + Strings.collectionToCommaDelimitedString(getSnapshotsResponse.getFailedResponses().keySet()) + + "] failed to retrieve snapshots"); + } + + for (Map.Entry> response : getSnapshotsResponse.getSuccessfulResponses().entrySet()) { + String repository = response.getKey(); + for (SnapshotInfo snapshotStatus : response.getValue()) { + table.startRow(); + + table.addCell(snapshotStatus.snapshotId().getName()); + table.addCell(repository); + table.addCell(snapshotStatus.state()); + table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.startTime(), TimeUnit.MILLISECONDS)); + table.addCell(FORMATTER.format(Instant.ofEpochMilli(snapshotStatus.startTime()))); + table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.endTime(), TimeUnit.MILLISECONDS)); + table.addCell(FORMATTER.format(Instant.ofEpochMilli(snapshotStatus.endTime()))); + final long durationMillis; + if (snapshotStatus.state() == SnapshotState.IN_PROGRESS) { + durationMillis = System.currentTimeMillis() - snapshotStatus.startTime(); + } else { + durationMillis = snapshotStatus.endTime() - snapshotStatus.startTime(); + } + table.addCell(TimeValue.timeValueMillis(durationMillis)); + table.addCell(snapshotStatus.indices().size()); + table.addCell(snapshotStatus.successfulShards()); + table.addCell(snapshotStatus.failedShards()); + table.addCell(snapshotStatus.totalShards()); + table.addCell(snapshotStatus.reason()); + + table.endRow(); } - table.addCell(TimeValue.timeValueMillis(durationMillis)); - table.addCell(snapshotStatus.indices().size()); - table.addCell(snapshotStatus.successfulShards()); - table.addCell(snapshotStatus.failedShards()); - table.addCell(snapshotStatus.totalShards()); - table.addCell(snapshotStatus.reason()); - - table.endRow(); } return table; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java index 7dac3a38dddd7..2ab9f7d89824b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java @@ -157,8 +157,8 @@ public void testGetSnapshotWithBlocks() { try { setClusterReadOnly(true); GetSnapshotsResponse response = client().admin().cluster().prepareGetSnapshots(REPOSITORY_NAME).execute().actionGet(); - assertThat(response.getSnapshots(), hasSize(1)); - assertThat(response.getSnapshots().get(0).snapshotId().getName(), equalTo(SNAPSHOT_NAME)); + assertThat(response.getSnapshots(REPOSITORY_NAME), hasSize(1)); + assertThat(response.getSnapshots(REPOSITORY_NAME).get(0).snapshotId().getName(), equalTo(SNAPSHOT_NAME)); } finally { setClusterReadOnly(false); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java index c5bd7d9f38ac1..5a70f7bf1ee96 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java @@ -19,44 +19,108 @@ package org.elasticsearch.action.admin.cluster.snapshots.get; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotShardFailure; -import org.elasticsearch.test.AbstractStreamableXContentTestCase; +import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; -public class GetSnapshotsResponseTests extends AbstractStreamableXContentTestCase { +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; +import static org.hamcrest.CoreMatchers.containsString; - @Override - protected GetSnapshotsResponse doParseInstance(XContentParser parser) throws IOException { +public class GetSnapshotsResponseTests extends ESTestCase { + // We can not subclass AbstractStreamableXContentTestCase because it + // can only be used for instances with equals and hashCode + // GetSnapshotResponse does not override equals and hashCode. + // It does not override equals and hashCode, because it + // contains ElasticsearchException, which does not override equals and hashCode. + + private GetSnapshotsResponse doParseInstance(XContentParser parser) throws IOException { return GetSnapshotsResponse.fromXContent(parser); } - @Override - protected GetSnapshotsResponse createBlankInstance() { - return new GetSnapshotsResponse(); + private GetSnapshotsResponse copyInstance(GetSnapshotsResponse instance, Version version) throws IOException { + return copyStreamable(instance, new NamedWriteableRegistry(Collections.emptyList()), () -> new GetSnapshotsResponse(), version); + } + + private void assertEqualInstances(GetSnapshotsResponse expectedInstance, GetSnapshotsResponse newInstance) { + assertEquals(expectedInstance.getSuccessfulResponses(), newInstance.getSuccessfulResponses()); + assertEquals(expectedInstance.getFailedResponses().keySet(), newInstance.getFailedResponses().keySet()); + for (Map.Entry expectedEntry : expectedInstance.getFailedResponses().entrySet()) { + ElasticsearchException expectedException = expectedEntry.getValue(); + ElasticsearchException newException = newInstance.getFailedResponses().get(expectedEntry.getKey()); + assertThat(newException.getMessage(), containsString(expectedException.getMessage())); + } } - @Override - protected GetSnapshotsResponse createTestInstance() { + private List createSnapshotInfos() { ArrayList snapshots = new ArrayList<>(); for (int i = 0; i < randomIntBetween(5, 10); ++i) { SnapshotId snapshotId = new SnapshotId("snapshot " + i, UUIDs.base64UUID()); String reason = randomBoolean() ? null : "reason"; ShardId shardId = new ShardId("index", UUIDs.base64UUID(), 2); List shardFailures = Collections.singletonList(new SnapshotShardFailure("node-id", shardId, "reason")); - snapshots.add(new SnapshotInfo(snapshotId, Arrays.asList("indice1", "indice2"), System.currentTimeMillis(), reason, - System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean())); + snapshots.add(new SnapshotInfo(snapshotId, Arrays.asList("index1", "index2"), System.currentTimeMillis(), reason, + System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean())); + + } + return snapshots; + } + + private GetSnapshotsResponse createTestInstance() { + Set repositories = new HashSet<>(); + Map> successfulResponses = new HashMap<>(); + Map failedResponses = new HashMap<>(); + for (int i = 0; i < randomIntBetween(0, 5); i++) { + String repository = randomValueOtherThanMany(r -> repositories.contains(r), () -> randomAlphaOfLength(10)); + repositories.add(repository); + successfulResponses.put(repository, createSnapshotInfos()); } - return new GetSnapshotsResponse(snapshots); + + for (int i = 0; i < randomIntBetween(0, 5); i++) { + String repository = randomValueOtherThanMany(r -> repositories.contains(r), () -> randomAlphaOfLength(10)); + repositories.add(repository); + failedResponses.put(repository, new ElasticsearchException(randomAlphaOfLength(10))); + } + + return new GetSnapshotsResponse(successfulResponses, failedResponses); + } + + public void testSerialization() throws IOException { + GetSnapshotsResponse testInstance = createTestInstance(); + GetSnapshotsResponse deserializedInstance = copyInstance(testInstance, Version.CURRENT); + assertEqualInstances(testInstance, deserializedInstance); + } + + public void testFromXContent() throws IOException { + xContentTester(this::createParser, this::createTestInstance, ToXContent.EMPTY_PARAMS, this::doParseInstance) + .numberOfTestRuns(1) + .supportsUnknownFields(true) + .shuffleFieldsExceptions(Strings.EMPTY_ARRAY) + .randomFieldsExcludeFilter(field -> false) + .assertEqualsConsumer(this::assertEqualInstances) + // We set it to false, because GetSnapshotsResponse contains + // ElasticsearchException, whose xContent creation/parsing are not stable. + .assertToXContentEquivalence(false) + .test(); } + } diff --git a/server/src/test/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java b/server/src/test/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java index 434c371278e06..e79bf35dda335 100644 --- a/server/src/test/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java @@ -230,7 +230,7 @@ public void testRestoreSnapshotOverLimit() { equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); List snapshotInfos = client.admin().cluster().prepareGetSnapshots("test-repo") - .setSnapshots("test-snap").get().getSnapshots(); + .setSnapshots("test-snap").get().getSnapshots("test-repo"); assertThat(snapshotInfos.size(), equalTo(1)); SnapshotInfo snapshotInfo = snapshotInfos.get(0); assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); diff --git a/server/src/test/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java index b9ac5f33dd911..969a5158cedc0 100644 --- a/server/src/test/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java @@ -179,7 +179,7 @@ private void assertAllSnapshotsCompleted() throws Exception { private void assertSnapshotExists(String repository, String snapshot) { GetSnapshotsResponse snapshotsStatusResponse = dataNodeClient().admin().cluster().prepareGetSnapshots(repository) .setSnapshots(snapshot).get(); - SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots(repository).get(0); assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); assertEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards()); assertEquals(0, snapshotInfo.failedShards()); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 4196472334ca9..c74aa89d2b2ef 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -512,7 +512,7 @@ public void testSnapshotRecovery() throws Exception { equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); assertThat(client().admin().cluster().prepareGetSnapshots(REPO_NAME).setSnapshots(SNAP_NAME).get() - .getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); + .getSnapshots(REPO_NAME).get(0).state(), equalTo(SnapshotState.SUCCESS)); client().admin().indices().prepareClose(INDEX_NAME).execute().actionGet(); diff --git a/server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 9fe7356877c8e..032e8a982ed4b 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -120,7 +120,7 @@ public SnapshotInfo waitForCompletion(String repository, String snapshotName, Ti long start = System.currentTimeMillis(); while (System.currentTimeMillis() - start < timeout.millis()) { List snapshotInfos = client().admin().cluster().prepareGetSnapshots(repository).setSnapshots(snapshotName) - .get().getSnapshots(); + .get().getSnapshots(repository); assertThat(snapshotInfos.size(), equalTo(1)); if (snapshotInfos.get(0).state().completed()) { // Make sure that snapshot clean up operations are finished diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 3345fbd3f248e..acd43ed732648 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -219,8 +219,7 @@ public void testExceptionWhenRestoringPersistentSettings() { assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0)); assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet() - .getSnapshots().get(0).state(), - equalTo(SnapshotState.SUCCESS)); + .getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS)); logger.info("--> change the test persistent setting and break it"); setSettingValue.accept("new value 2"); @@ -272,7 +271,7 @@ public void testRestoreCustomMetadata() throws Exception { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards())); assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet() - .getSnapshots().get(0).state(), + .getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS)); logger.info("--> change custom persistent metadata"); @@ -584,8 +583,8 @@ public void testRestoreIndexWithMissingShards() throws Exception { assertBusy(() -> { GetSnapshotsResponse response = client().admin().cluster().prepareGetSnapshots("test-repo") .setSnapshots("test-snap-2").get(); - assertThat(response.getSnapshots().size(), equalTo(1)); - SnapshotInfo snapshotInfo = response.getSnapshots().get(0); + assertThat(response.getSnapshots("test-repo").size(), equalTo(1)); + SnapshotInfo snapshotInfo = response.getSnapshots("test-repo").get(0); assertTrue(snapshotInfo.state().completed()); assertEquals(SnapshotState.PARTIAL, snapshotInfo.state()); }, 1, TimeUnit.MINUTES); @@ -600,7 +599,7 @@ public void testRestoreIndexWithMissingShards() throws Exception { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), lessThan(16)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(10)); assertThat(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-2").execute().actionGet() - .getSnapshots().get(0).state(), + .getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.PARTIAL)); } @@ -841,7 +840,7 @@ public void testMasterShutdownDuringSnapshot() throws Exception { assertBusy(() -> { GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo") .setSnapshots("test-snap").get(); - SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0); assertTrue(snapshotInfo.state().completed()); }, 1, TimeUnit.MINUTES); @@ -849,7 +848,7 @@ public void testMasterShutdownDuringSnapshot() throws Exception { GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo") .setSnapshots("test-snap").get(); - SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0); assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); assertEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards()); assertEquals(0, snapshotInfo.failedShards()); @@ -901,7 +900,7 @@ public void testMasterAndDataShutdownDuringSnapshot() throws Exception { assertBusy(() -> { GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo") .setSnapshots("test-snap").get(); - SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0); assertTrue(snapshotInfo.state().completed()); }, 1, TimeUnit.MINUTES); @@ -909,7 +908,7 @@ public void testMasterAndDataShutdownDuringSnapshot() throws Exception { GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo") .setSnapshots("test-snap").get(); - SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0); assertEquals(SnapshotState.PARTIAL, snapshotInfo.state()); assertNotEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards()); assertThat(snapshotInfo.failedShards(), greaterThan(0)); @@ -967,8 +966,8 @@ public void testMasterShutdownDuringFailedSnapshot() throws Exception { assertBusy(() -> { GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster() .prepareGetSnapshots("test-repo").setSnapshots("test-snap").setIgnoreUnavailable(true).get(); - assertEquals(1, snapshotsStatusResponse.getSnapshots().size()); - SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0); + assertEquals(1, snapshotsStatusResponse.getSnapshots("test-repo").size()); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0); assertTrue(snapshotInfo.state().completed()); ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); SnapshotsInProgress snapshotsInProgress = clusterState.custom(SnapshotsInProgress.TYPE); @@ -978,7 +977,7 @@ public void testMasterShutdownDuringFailedSnapshot() throws Exception { logger.info("--> verify that snapshot failed"); GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster() .prepareGetSnapshots("test-repo").setSnapshots("test-snap").get(); - SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0); assertEquals(SnapshotState.FAILED, snapshotInfo.state()); } @@ -1216,8 +1215,8 @@ public void testDataNodeRestartWithBusyMasterDuringSnapshot() throws Exception { assertBusy(() -> { GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster() .prepareGetSnapshots("test-repo").setSnapshots("test-snap").setIgnoreUnavailable(true).get(); - assertEquals(1, snapshotsStatusResponse.getSnapshots().size()); - SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0); + assertEquals(1, snapshotsStatusResponse.getSnapshots("test-repo").size()); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0); assertTrue(snapshotInfo.state().toString(), snapshotInfo.state().completed()); }, 60L, TimeUnit.SECONDS); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java index aeea321b8536b..367dc44a9a686 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java @@ -89,7 +89,7 @@ public void testWhenMetadataAreLoaded() throws Exception { // Getting a snapshot does not load any metadata GetSnapshotsResponse getSnapshotsResponse = client().admin().cluster().prepareGetSnapshots("repository").addSnapshots("snap").setVerbose(randomBoolean()).get(); - assertThat(getSnapshotsResponse.getSnapshots(), hasSize(1)); + assertThat(getSnapshotsResponse.getSnapshots("repository"), hasSize(1)); assertGlobalMetadataLoads("snap", 0); assertIndexMetadataLoads("snap", "docs", 0); assertIndexMetadataLoads("snap", "others", 0); diff --git a/server/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java index 885baa883ed63..0dd8d6af585d8 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java @@ -104,7 +104,7 @@ public void testConcurrentSnapshotDeletionsNotAllowed() throws Exception { logger.info("--> delete second snapshot, which should now work"); client().admin().cluster().prepareDeleteSnapshot(repo, snapshot1).get(); - assertTrue(client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots().isEmpty()); + assertTrue(client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots(repo).isEmpty()); } public void testSnapshottingWithInProgressDeletionNotAllowed() throws Exception { @@ -150,7 +150,7 @@ public void testSnapshottingWithInProgressDeletionNotAllowed() throws Exception logger.info("--> creating second snapshot, which should now work"); client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get(); - assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots().size()); + assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots(repo).size()); } public void testRestoreWithInProgressDeletionsNotAllowed() throws Exception { @@ -204,6 +204,6 @@ public void testRestoreWithInProgressDeletionsNotAllowed() throws Exception { logger.info("--> restoring snapshot, which should now work"); client().admin().cluster().prepareRestoreSnapshot(repo, snapshot1).setWaitForCompletion(true).get(); - assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots().size()); + assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots(repo).size()); } } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 3a78b4786fc5c..6ac486b51e912 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -116,6 +116,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -239,7 +240,7 @@ public void testBasicWorkFlow() throws Exception { equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); List snapshotInfos = client.admin().cluster().prepareGetSnapshots("test-repo") - .setSnapshots(randomFrom("test-snap", "_all", "*", "*-snap", "test*")).get().getSnapshots(); + .setSnapshots(randomFrom("test-snap", "_all", "*", "*-snap", "test*")).get().getSnapshots("test-repo"); assertThat(snapshotInfos.size(), equalTo(1)); SnapshotInfo snapshotInfo = snapshotInfos.get(0); assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); @@ -478,8 +479,8 @@ public void testEmptySnapshot() throws Exception { assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0)); - assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), - equalTo(SnapshotState.SUCCESS)); + assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap") + .get().getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS)); } public void testRestoreAliases() throws Exception { @@ -576,8 +577,8 @@ public void testRestoreTemplates() throws Exception { .setIndices().setWaitForCompletion(true).get(); assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0)); - assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), - equalTo(SnapshotState.SUCCESS)); + assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get(). + getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS)); logger.info("--> delete test template"); assertThat(client.admin().indices().prepareDeleteTemplate("test-template").get().isAcknowledged(), equalTo(true)); @@ -660,7 +661,7 @@ public void testIncludeGlobalState() throws Exception { assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0)); assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-no-global-state") - .get().getSnapshots().get(0).state(), + .get().getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS)); SnapshotsStatusResponse snapshotsStatusResponse = client.admin().cluster().prepareSnapshotStatus("test-repo") .addSnapshots("test-snap-no-global-state").get(); @@ -674,7 +675,7 @@ public void testIncludeGlobalState() throws Exception { assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0)); assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-with-global-state") - .get().getSnapshots().get(0).state(), + .get().getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS)); snapshotsStatusResponse = client.admin().cluster().prepareSnapshotStatus("test-repo") .addSnapshots("test-snap-with-global-state").get(); @@ -749,7 +750,7 @@ public void testIncludeGlobalState() throws Exception { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-no-global-state-with-index") - .get().getSnapshots().get(0).state(), + .get().getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS)); logger.info("--> delete global state and index "); @@ -822,8 +823,8 @@ public void testSnapshotFileFailureDuringSnapshot() { } GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("test-repo") .addSnapshots("test-snap").get(); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1)); - SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots().get(0); + assertThat(getSnapshotsResponse.getSnapshots("test-repo").size(), equalTo(1)); + SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots("test-repo").get(0); if (snapshotInfo.state() == SnapshotState.SUCCESS) { assertThat(snapshotInfo.shardFailures().size(), greaterThan(0)); assertThat(snapshotInfo.totalShards(), greaterThan(snapshotInfo.successfulShards())); @@ -878,8 +879,8 @@ public void testDataFileFailureDuringSnapshot() throws Exception { } GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("test-repo") .addSnapshots("test-snap").get(); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1)); - SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots().get(0); + assertThat(getSnapshotsResponse.getSnapshots("test-repo").size(), equalTo(1)); + SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots("test-repo").get(0); assertThat(snapshotInfo.state(), equalTo(SnapshotState.PARTIAL)); assertThat(snapshotInfo.shardFailures().size(), greaterThan(0)); assertThat(snapshotInfo.totalShards(), greaterThan(snapshotInfo.successfulShards())); @@ -1322,6 +1323,96 @@ public void testDeleteSnapshot() throws Exception { assertThat(numberOfFiles(repo), equalTo(numberOfFiles[0] + 2)); } + public void testGetSnapshotsMultipleRepos() { + final Client client = client(); + + List snapshotList = new ArrayList<>(); + List repoList = new ArrayList<>(); + Map> repo2SnapshotNames = new HashMap<>(); + + logger.info("--> create an index and index some documents"); + final String indexName = "test-idx"; + assertAcked(prepareCreate(indexName)); + ensureGreen(); + for (int i = 0; i < 10; i++) { + index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + for (int repoIndex = 0; repoIndex < randomIntBetween(2, 5); repoIndex++) { + final String repoName = "repo" + repoIndex; + repoList.add(repoName); + final Path repoPath = randomRepoPath(); + logger.info("--> create repository with name " + repoName); + assertAcked(client.admin().cluster().preparePutRepository(repoName) + .setType("mock").setSettings(Settings.builder() + .put("location", repoPath) + .put("compress", false) + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put("wait_after_unblock", 200))); + List snapshotNames = new ArrayList<>(); + repo2SnapshotNames.put(repoName, snapshotNames); + + for (int snapshotIndex = 0; snapshotIndex < randomIntBetween(2, 5); snapshotIndex++) { + final String snapshotName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + snapshotList.add(snapshotName); + logger.info("--> create snapshot with index {} and name {} in repository {}", snapshotIndex, snapshotName, repoName); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true) + .setIndices(indexName) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + snapshotNames.add(snapshotName); + } + } + + Supplier repoNames = () -> randomFrom(new String[]{"_all"}, + new String[]{"repo*"}, repoList.toArray(new String[0])); + + + logger.info("--> get and verify snapshots"); + GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster() + .prepareGetSnapshots(repoNames.get()) + .setSnapshots(randomFrom("_all", "*")) + .get(); + + for (Map.Entry> repo2Names : repo2SnapshotNames.entrySet()) { + String repo = repo2Names.getKey(); + List snapshotNames = repo2Names.getValue(); + List snapshots = getSnapshotsResponse.getSnapshots(repo); + assertEquals(snapshotNames, snapshots.stream().map(s -> s.snapshotId().getName()).collect(Collectors.toList())); + } + + + logger.info("--> specify all snapshot names with ignoreUnavailable=false"); + GetSnapshotsResponse getSnapshotsResponse2 = client.admin().cluster() + .prepareGetSnapshots(randomFrom("_all", "repo*")) + .setIgnoreUnavailable(false) + .setSnapshots(snapshotList.toArray(new String[0])) + .get(); + + for (String repo : repoList) { + expectThrows(SnapshotMissingException.class, () -> getSnapshotsResponse2.getSnapshots(repo)); + } + + + logger.info("--> specify all snapshot names with ignoreUnavailable=true"); + GetSnapshotsResponse getSnapshotsResponse3 = client.admin().cluster() + .prepareGetSnapshots(randomFrom("_all", "repo*")) + .setIgnoreUnavailable(true) + .setSnapshots(snapshotList.toArray(new String[0])) + .get(); + + for (Map.Entry> repo2Names : repo2SnapshotNames.entrySet()) { + String repo = repo2Names.getKey(); + List snapshotNames = repo2Names.getValue(); + List snapshots = getSnapshotsResponse3.getSnapshots(repo); + assertEquals(snapshotNames, snapshots.stream().map(s -> s.snapshotId().getName()).collect(Collectors.toList())); + } + } + public void testDeleteSnapshotWithMissingIndexAndShardMetadata() throws Exception { Client client = client(); @@ -1366,8 +1457,9 @@ public void testDeleteSnapshotWithMissingIndexAndShardMetadata() throws Exceptio client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get(); logger.info("--> make sure snapshot doesn't exist"); - assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo") - .addSnapshots("test-snap-1"), SnapshotMissingException.class); + + expectThrows(SnapshotMissingException.class, () -> client.admin().cluster().prepareGetSnapshots("test-repo") + .addSnapshots("test-snap-1").get().getSnapshots("test-repo")); for (String index : indices) { assertTrue(Files.notExists(indicesPath.resolve(indexIds.get(index).getId()))); @@ -1406,8 +1498,8 @@ public void testDeleteSnapshotWithMissingMetadata() throws Exception { client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get(); logger.info("--> make sure snapshot doesn't exist"); - assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo") - .addSnapshots("test-snap-1"), SnapshotMissingException.class); + expectThrows(SnapshotMissingException.class, () -> client.admin().cluster().prepareGetSnapshots("test-repo") + .addSnapshots("test-snap-1").get().getSnapshots("test-repo")); } public void testDeleteSnapshotWithCorruptedSnapshotFile() throws Exception { @@ -1443,7 +1535,9 @@ public void testDeleteSnapshotWithCorruptedSnapshotFile() throws Exception { client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get(); logger.info("--> make sure snapshot doesn't exist"); - assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1"), SnapshotMissingException.class); + expectThrows(SnapshotMissingException.class, + () -> client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1").get(). + getSnapshots("test-repo")); logger.info("--> make sure that we can create the snapshot again"); createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1") @@ -1489,7 +1583,7 @@ public void testDeleteSnapshotWithCorruptedGlobalState() throws Exception { } } - List snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots(); + List snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots("test-repo"); assertThat(snapshotInfos.size(), equalTo(1)); assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS)); assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap")); @@ -1500,8 +1594,8 @@ public void testDeleteSnapshotWithCorruptedGlobalState() throws Exception { assertThat(snapshotStatusResponse.getSnapshots().get(0).getSnapshot().getSnapshotId().getName(), equalTo("test-snap")); assertAcked(client().admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get()); - assertThrows(client().admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap"), - SnapshotMissingException.class); + expectThrows(SnapshotMissingException.class, () -> client().admin().cluster() + .prepareGetSnapshots("test-repo").addSnapshots("test-snap").get().getSnapshots("test-repo")); assertThrows(client().admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap"), SnapshotMissingException.class); @@ -1748,7 +1842,7 @@ public void testMoveShardWhileSnapshotting() throws Exception { logger.info("--> done"); List snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo") - .setSnapshots("test-snap").get().getSnapshots(); + .setSnapshots("test-snap").get().getSnapshots("test-repo"); assertThat(snapshotInfos.size(), equalTo(1)); assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS)); @@ -1833,7 +1927,7 @@ public void testDeleteRepositoryWhileSnapshotting() throws Exception { logger.info("--> done"); List snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo") - .setSnapshots("test-snap").get().getSnapshots(); + .setSnapshots("test-snap").get().getSnapshots("test-repo"); assertThat(snapshotInfos.size(), equalTo(1)); assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS)); @@ -1882,8 +1976,8 @@ public void testReadonlyRepository() throws Exception { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); - assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), - equalTo(SnapshotState.SUCCESS)); + assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get() + .getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS)); logger.info("--> delete index"); cluster().wipeIndices("test-idx"); @@ -1904,8 +1998,8 @@ public void testReadonlyRepository() throws Exception { logger.info("--> list available shapshots"); GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("readonly-repo").get(); - assertThat(getSnapshotsResponse.getSnapshots(), notNullValue()); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1)); + assertThat(getSnapshotsResponse.getSnapshots("readonly-repo"), notNullValue()); + assertThat(getSnapshotsResponse.getSnapshots("readonly-repo").size(), equalTo(1)); logger.info("--> try deleting snapshot"); assertThrows(client.admin().cluster().prepareDeleteSnapshot("readonly-repo", "test-snap"), RepositoryException.class, @@ -2046,8 +2140,8 @@ public void testSnapshotStatus() throws Exception { logger.info("--> checking that _current returns the currently running snapshot"); GetSnapshotsResponse getResponse = client.admin().cluster().prepareGetSnapshots("test-repo") .setCurrentSnapshot().execute().actionGet(); - assertThat(getResponse.getSnapshots().size(), equalTo(1)); - SnapshotInfo snapshotInfo = getResponse.getSnapshots().get(0); + assertThat(getResponse.getSnapshots("test-repo").size(), equalTo(1)); + SnapshotInfo snapshotInfo = getResponse.getSnapshots("test-repo").get(0); assertThat(snapshotInfo.state(), equalTo(SnapshotState.IN_PROGRESS)); logger.info("--> unblocking blocked node"); @@ -2077,7 +2171,7 @@ public void testSnapshotStatus() throws Exception { logger.info("--> checking that _current no longer returns the snapshot"); assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("_current") - .execute().actionGet().getSnapshots().isEmpty(), + .execute().actionGet().getSnapshots("test-repo").isEmpty(), equalTo(true)); // test that getting an unavailable snapshot status throws an exception if ignoreUnavailable is false on the request @@ -2167,8 +2261,8 @@ public void testSnapshotMoreThanOnce() throws ExecutionException, InterruptedExc assertThat(createSnapshotResponseFirst.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponseFirst.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponseFirst.getSnapshotInfo().totalShards())); - assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test").get().getSnapshots().get(0).state(), - equalTo(SnapshotState.SUCCESS)); + assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test").get() + .getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS)); { SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo") .setSnapshots("test").get().getSnapshots().get(0); @@ -2183,8 +2277,8 @@ public void testSnapshotMoreThanOnce() throws ExecutionException, InterruptedExc assertThat(createSnapshotResponseSecond.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponseSecond.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponseSecond.getSnapshotInfo().totalShards())); - assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-1").get().getSnapshots().get(0).state(), - equalTo(SnapshotState.SUCCESS)); + assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-1").get() + .getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS)); { SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo") .setSnapshots("test-1").get().getSnapshots().get(0); @@ -2200,8 +2294,8 @@ public void testSnapshotMoreThanOnce() throws ExecutionException, InterruptedExc assertThat(createSnapshotResponseThird.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponseThird.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponseThird.getSnapshotInfo().totalShards())); - assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-2").get().getSnapshots().get(0).state(), - equalTo(SnapshotState.SUCCESS)); + assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-2").get() + .getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS)); { SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo") .setSnapshots("test-2").get().getSnapshots().get(0); @@ -2757,7 +2851,8 @@ public void testSnapshotName() throws Exception { expectThrows(InvalidSnapshotNameException.class, () -> client.admin().cluster().prepareCreateSnapshot("test-repo", "_foo").get()); expectThrows(SnapshotMissingException.class, - () -> client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("_foo").get()); + () -> client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("_foo") + .get().getSnapshots("test-repo")); expectThrows(SnapshotMissingException.class, () -> client.admin().cluster().prepareDeleteSnapshot("test-repo", "_foo").get()); expectThrows(SnapshotMissingException.class, @@ -2802,14 +2897,14 @@ public void testListCorruptedSnapshot() throws Exception { logger.info("--> get snapshots request should return both snapshots"); List snapshotInfos = client.admin().cluster() .prepareGetSnapshots("test-repo") - .setIgnoreUnavailable(true).get().getSnapshots(); + .setIgnoreUnavailable(true).get().getSnapshots("test-repo"); assertThat(snapshotInfos.size(), equalTo(1)); assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS)); assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap-1")); final SnapshotException ex = expectThrows(SnapshotException.class, () -> - client.admin().cluster().prepareGetSnapshots("test-repo").setIgnoreUnavailable(false).get()); + client.admin().cluster().prepareGetSnapshots("test-repo").setIgnoreUnavailable(false).get().getSnapshots("test-repo")); assertThat(ex.getRepositoryName(), equalTo("test-repo")); assertThat(ex.getSnapshotName(), equalTo("test-snap-2")); } @@ -2844,7 +2939,7 @@ public void testRestoreSnapshotWithCorruptedGlobalState() throws Exception { outChan.truncate(randomInt(10)); } - List snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots(); + List snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots("test-repo"); assertThat(snapshotInfos.size(), equalTo(1)); assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS)); assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap")); @@ -2934,7 +3029,7 @@ public void testRestoreSnapshotWithCorruptedIndexMetadata() throws Exception { outChan.truncate(randomInt(10)); } - List snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots(); + List snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots("test-repo"); assertThat(snapshotInfos.size(), equalTo(1)); assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS)); assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap")); @@ -3013,7 +3108,7 @@ public void testSnapshotWithCorruptedShardIndexFile() throws Exception { } logger.info("--> verifying snapshot state for [{}]", snapshot1); - List snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots(); + List snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots("test-repo"); assertThat(snapshotInfos.size(), equalTo(1)); assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS)); assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo(snapshot1)); @@ -3145,7 +3240,8 @@ public void testGetSnapshotsRequest() throws Exception { .cluster() .prepareGetSnapshots(repositoryName) .addSnapshots("non-existent-snapshot") - .get()); + .get() + .getSnapshots(repositoryName)); // with ignore unavailable set to true, should not throw an exception GetSnapshotsResponse getSnapshotsResponse = client.admin() .cluster() @@ -3153,7 +3249,7 @@ public void testGetSnapshotsRequest() throws Exception { .setIgnoreUnavailable(true) .addSnapshots("non-existent-snapshot") .get(); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0)); + assertThat(getSnapshotsResponse.getSnapshots(repositoryName).size(), equalTo(0)); logger.info("--> creating an index and indexing documents"); // Create index on 2 nodes and make sure each node has a primary by setting no replicas @@ -3178,8 +3274,8 @@ public void testGetSnapshotsRequest() throws Exception { .prepareGetSnapshots("test-repo") .setSnapshots(randomFrom("_all", "_current", "snap-on-*", "*-on-empty-repo", "snap-on-empty-repo")) .get(); - assertEquals(1, getSnapshotsResponse.getSnapshots().size()); - assertEquals("snap-on-empty-repo", getSnapshotsResponse.getSnapshots().get(0).snapshotId().getName()); + assertEquals(1, getSnapshotsResponse.getSnapshots("test-repo").size()); + assertEquals("snap-on-empty-repo", getSnapshotsResponse.getSnapshots("test-repo").get(0).snapshotId().getName()); unblockNode(repositoryName, initialBlockedNode); // unblock node responseListener.actionGet(TimeValue.timeValueMillis(10000L)); // timeout after 10 seconds client.admin().cluster().prepareDeleteSnapshot(repositoryName, "snap-on-empty-repo").get(); @@ -3233,8 +3329,8 @@ public void testGetSnapshotsRequest() throws Exception { .get(); List sortedNames = Arrays.asList(snapshotNames); Collections.sort(sortedNames); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(numSnapshots)); - assertThat(getSnapshotsResponse.getSnapshots().stream() + assertThat(getSnapshotsResponse.getSnapshots(repositoryName).size(), equalTo(numSnapshots)); + assertThat(getSnapshotsResponse.getSnapshots(repositoryName).stream() .map(s -> s.snapshotId().getName()) .sorted() .collect(Collectors.toList()), equalTo(sortedNames)); @@ -3245,8 +3341,8 @@ public void testGetSnapshotsRequest() throws Exception { .get(); sortedNames = Arrays.asList(snapshotNames); Collections.sort(sortedNames); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(numSnapshots)); - assertThat(getSnapshotsResponse.getSnapshots().stream() + assertThat(getSnapshotsResponse.getSnapshots(repositoryName).size(), equalTo(numSnapshots)); + assertThat(getSnapshotsResponse.getSnapshots(repositoryName).stream() .map(s -> s.snapshotId().getName()) .sorted() .collect(Collectors.toList()), equalTo(sortedNames)); @@ -3261,8 +3357,8 @@ public void testGetSnapshotsRequest() throws Exception { .addSnapshots(snapshotNames) .addSnapshots(firstRegex, secondRegex) .get(); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(numSnapshots)); - assertThat(getSnapshotsResponse.getSnapshots().stream() + assertThat(getSnapshotsResponse.getSnapshots(repositoryName).size(), equalTo(numSnapshots)); + assertThat(getSnapshotsResponse.getSnapshots(repositoryName).stream() .map(s -> s.snapshotId().getName()) .sorted() .collect(Collectors.toList()), equalTo(sortedNames)); @@ -3388,7 +3484,7 @@ public void testSnapshotSucceedsAfterSnapshotFailure() throws Exception { assertEquals(0, createSnapshotResponse.getSnapshotInfo().failedShards()); GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("test-repo-2") .addSnapshots("test-snap-2").get(); - assertEquals(SnapshotState.SUCCESS, getSnapshotsResponse.getSnapshots().get(0).state()); + assertEquals(SnapshotState.SUCCESS, getSnapshotsResponse.getSnapshots("test-repo-2").get(0).state()); } public void testSnapshotStatusOnFailedIndex() throws Exception { @@ -3517,8 +3613,8 @@ public void testGetSnapshotsFromIndexBlobOnly() throws Exception { .setSnapshots("_all") .setVerbose(false) .get(); - assertEquals(indicesPerSnapshot.size(), response.getSnapshots().size()); - verifySnapshotInfo(response, indicesPerSnapshot); + assertEquals(indicesPerSnapshot.size(), response.getSnapshots("test-repo").size()); + verifySnapshotInfo("test-repo", response, indicesPerSnapshot); logger.info("--> verify wildcard returns snapshot info"); response = client().admin().cluster() @@ -3526,8 +3622,8 @@ public void testGetSnapshotsFromIndexBlobOnly() throws Exception { .setSnapshots("test-snap-*") .setVerbose(false) .get(); - assertEquals(indicesPerSnapshot.size(), response.getSnapshots().size()); - verifySnapshotInfo(response, indicesPerSnapshot); + assertEquals(indicesPerSnapshot.size(), response.getSnapshots("test-repo").size()); + verifySnapshotInfo("test-repo", response, indicesPerSnapshot); logger.info("--> verify individual requests return snapshot info"); for (int i = 0; i < numSnapshots; i++) { @@ -3536,8 +3632,8 @@ public void testGetSnapshotsFromIndexBlobOnly() throws Exception { .setSnapshots("test-snap-" + i) .setVerbose(false) .get(); - assertEquals(1, response.getSnapshots().size()); - verifySnapshotInfo(response, indicesPerSnapshot); + assertEquals(1, response.getSnapshots("test-repo").size()); + verifySnapshotInfo("test-repo", response, indicesPerSnapshot); } } @@ -3792,7 +3888,8 @@ public void testAbortedSnapshotDuringInitDoesNotStart() throws Exception { assertAcked(delete.get()); expectThrows(SnapshotMissingException.class, () -> - client.admin().cluster().prepareGetSnapshots("repository").setSnapshots("snap").get()); + client.admin().cluster().prepareGetSnapshots("repository").setSnapshots("snap").get() + .getSnapshots("repository")); assertFalse("Expecting snapshot state to be updated", states.isEmpty()); assertFalse("Expecting snapshot to be aborted and not started at all", states.contains(State.STARTED)); @@ -3859,8 +3956,9 @@ private RepositoryData getRepositoryData(Repository repository) throws Interrupt return repositoryData.get(); } - private void verifySnapshotInfo(final GetSnapshotsResponse response, final Map> indicesPerSnapshot) { - for (SnapshotInfo snapshotInfo : response.getSnapshots()) { + private void verifySnapshotInfo(final String repo, final GetSnapshotsResponse response, + final Map> indicesPerSnapshot) { + for (SnapshotInfo snapshotInfo : response.getSnapshots("test-repo")) { final List expected = snapshotInfo.indices(); assertEquals(expected, indicesPerSnapshot.get(snapshotInfo.snapshotId().getName())); assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java index 777918a7d5eba..9f9eb8fe94206 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java @@ -77,7 +77,7 @@ public void testRetryPostingSnapshotStatusMessages() throws Exception { waitForBlock(blockedNode, "test-repo", TimeValue.timeValueSeconds(60)); final SnapshotId snapshotId = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap") - .get().getSnapshots().get(0).snapshotId(); + .get().getSnapshots("test-repo").get(0).snapshotId(); logger.info("--> start disrupting cluster"); final NetworkDisruption networkDisruption = new NetworkDisruption(new NetworkDisruption.TwoPartitions(masterNode, dataNode), @@ -106,7 +106,7 @@ public void testRetryPostingSnapshotStatusMessages() throws Exception { GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster() .prepareGetSnapshots("test-repo") .setSnapshots("test-snap").get(); - SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0); logger.info("Snapshot status [{}], successfulShards [{}]", snapshotInfo.state(), snapshotInfo.successfulShards()); assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); assertThat(snapshotInfo.successfulShards(), equalTo(shards)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 87596beb451d1..e7fc73fe985b9 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -540,7 +540,9 @@ private void wipeCluster() throws Exception { // All other repo types we really don't have a chance of being able to iterate properly, sadly. Request listRequest = new Request("GET", "/_snapshot/" + repoName + "/_all"); listRequest.addParameter("ignore_unavailable", "true"); - List snapshots = (List) entityAsMap(adminClient.performRequest(listRequest)).get("snapshots"); + List snapshots = + (List)((Map)((List) entityAsMap(adminClient.performRequest(listRequest)).get("responses")).get(0)) + .get("snapshots"); for (Object snapshot : snapshots) { Map snapshotInfo = (Map) snapshot; String name = (String) snapshotInfo.get("snapshot"); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java index 66ea23b518c29..a2e9aebff015b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java @@ -75,9 +75,10 @@ public void testSnapshotUserRoleCanSnapshotAndSeeAllIndices() { assertThat(snapshotResponse.getSnapshotInfo().indices(), containsInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7, ordinaryIndex)); // view snapshots for repo final GetSnapshotsResponse getSnapshotResponse = client.admin().cluster().prepareGetSnapshots("repo").get(); - assertThat(getSnapshotResponse.getSnapshots().size(), is(1)); - assertThat(getSnapshotResponse.getSnapshots().get(0).snapshotId().getName(), is("snap")); - assertThat(getSnapshotResponse.getSnapshots().get(0).indices(), containsInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7, ordinaryIndex)); + assertThat(getSnapshotResponse.getSnapshots("repo").size(), is(1)); + assertThat(getSnapshotResponse.getSnapshots("repo").get(0).snapshotId().getName(), is("snap")); + assertThat(getSnapshotResponse.getSnapshots("repo").get(0).indices(), containsInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7, + ordinaryIndex)); } public void testSnapshotUserRoleIsReserved() { From c2e24f2a3728c082c199e4e96aff9cefc35ae444 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 23 May 2019 16:26:45 -0400 Subject: [PATCH 02/35] Run on GENERIC thread pool --- .../cluster/snapshots/get/TransportGetSnapshotsAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 1efd0497dd845..4cef566ac2df1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -108,9 +108,9 @@ private void getMultipleReposSnapshotInfo(List repos, String boolean verbose, ActionListener listener) { List>> futures = new ArrayList<>(repos.size()); - // run concurrently for all repos on SNAPSHOT thread pool + // run concurrently for all repos on GENERIC thread pool for (final RepositoryMetaData repo : repos) { - futures.add(threadPool.executor(ThreadPool.Names.SNAPSHOT).submit( + futures.add(threadPool.executor(ThreadPool.Names.GENERIC).submit( () -> getSingleRepoSnapshotInfo(repo.name(), snapshots, ignoreUnavailable, verbose))); } assert repos.size() == futures.size(); From 94e60ddb60c0ccf551ff78f498e80dc260e14297 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 23 May 2019 17:13:51 -0400 Subject: [PATCH 03/35] Use GroupedActionListener --- .../get/TransportGetSnapshotsAction.java | 65 +++++++++++-------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 4cef566ac2df1..384004016d83c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.admin.cluster.snapshots.get; -import org.apache.logging.log4j.core.util.Throwables; import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; @@ -28,6 +27,7 @@ import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -35,6 +35,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.repositories.IndexId; @@ -50,11 +51,10 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; import java.util.stream.Collectors; /** @@ -106,34 +106,45 @@ protected void masterOperation(final GetSnapshotsRequest request, final ClusterS private void getMultipleReposSnapshotInfo(List repos, String[] snapshots, boolean ignoreUnavailable, boolean verbose, ActionListener listener) { - List>> futures = new ArrayList<>(repos.size()); + GroupedActionListener, ElasticsearchException>>> groupedActionListener = + new GroupedActionListener<>( + ActionListener.map(listener, responses -> { + assert repos.size() == responses.size(); + + Map> successfulResponses = new HashMap<>(); + Map failedResponses = new HashMap<>(); + + Iterator, ElasticsearchException>>> it = responses.iterator(); + + while (it.hasNext()) { + Tuple, ElasticsearchException>> response = it.next(); + String repo = response.v1(); + Tuple, ElasticsearchException> result = response.v2(); + if (result.v1() != null) { + assert result.v2() == null; + successfulResponses.put(repo, result.v1()); + } else { + assert result.v2() != null; + failedResponses.put(repo, result.v2()); + } + } + + return new GetSnapshotsResponse(successfulResponses, failedResponses); + }), repos.size()); // run concurrently for all repos on GENERIC thread pool for (final RepositoryMetaData repo : repos) { - futures.add(threadPool.executor(ThreadPool.Names.GENERIC).submit( - () -> getSingleRepoSnapshotInfo(repo.name(), snapshots, ignoreUnavailable, verbose))); - } - assert repos.size() == futures.size(); - - Map> successfulResponses = new HashMap<>(); - Map failedResponses = new HashMap<>(); - - for (int i = 0; i < repos.size(); i++) { - final String repo = repos.get(i).name(); - try { - successfulResponses.put(repo, futures.get(i).get()); - } catch (InterruptedException e) { - Throwables.rethrow(e); - } catch (ExecutionException e) { - if (e.getCause() instanceof ElasticsearchException) { - failedResponses.put(repo, (ElasticsearchException) e.getCause()); - } else { - Throwables.rethrow(e); - } - } + threadPool.executor(ThreadPool.Names.GENERIC).execute( + () -> { + // Unfortunately, there is no Either in Java, so we use Tuple with only one value set + try { + groupedActionListener.onResponse(Tuple.tuple(repo.name(), + Tuple.tuple(getSingleRepoSnapshotInfo(repo.name(), snapshots, ignoreUnavailable, verbose), null))); + } catch (ElasticsearchException e) { + groupedActionListener.onResponse(Tuple.tuple(repo.name(), Tuple.tuple(null, e))); + } + }); } - - listener.onResponse(new GetSnapshotsResponse(successfulResponses, failedResponses)); } private List getSingleRepoSnapshotInfo(String repo, String[] snapshots, boolean ignoreUnavailable, boolean verbose) { From 3bbea578ba9d5604acebd7a75e4b2c0c26cc3400 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 23 May 2019 17:19:49 -0400 Subject: [PATCH 04/35] Use fs repository --- .../snapshots/SharedClusterSnapshotRestoreIT.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 6ac486b51e912..8e97521fc3260 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -1345,11 +1345,7 @@ public void testGetSnapshotsMultipleRepos() { final Path repoPath = randomRepoPath(); logger.info("--> create repository with name " + repoName); assertAcked(client.admin().cluster().preparePutRepository(repoName) - .setType("mock").setSettings(Settings.builder() - .put("location", repoPath) - .put("compress", false) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES) - .put("wait_after_unblock", 200))); + .setType("fs").setSettings(Settings.builder().put("location", repoPath).build())); List snapshotNames = new ArrayList<>(); repo2SnapshotNames.put(repoName, snapshotNames); From 94f4d6303d78f444d33b5353d417de576c3979dc Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 23 May 2019 17:21:36 -0400 Subject: [PATCH 05/35] Remove supplier --- .../snapshots/SharedClusterSnapshotRestoreIT.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 8e97521fc3260..3dc1a4571f360 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -1364,13 +1364,9 @@ public void testGetSnapshotsMultipleRepos() { } } - Supplier repoNames = () -> randomFrom(new String[]{"_all"}, - new String[]{"repo*"}, repoList.toArray(new String[0])); - - logger.info("--> get and verify snapshots"); GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster() - .prepareGetSnapshots(repoNames.get()) + .prepareGetSnapshots(randomFrom(new String[]{"_all"}, new String[]{"repo*"}, repoList.toArray(new String[0]))) .setSnapshots(randomFrom("_all", "*")) .get(); @@ -1381,7 +1377,6 @@ public void testGetSnapshotsMultipleRepos() { assertEquals(snapshotNames, snapshots.stream().map(s -> s.snapshotId().getName()).collect(Collectors.toList())); } - logger.info("--> specify all snapshot names with ignoreUnavailable=false"); GetSnapshotsResponse getSnapshotsResponse2 = client.admin().cluster() .prepareGetSnapshots(randomFrom("_all", "repo*")) From 845ed5f7e93bae2876c5bb5f80a809e559bb30c7 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 23 May 2019 17:26:34 -0400 Subject: [PATCH 06/35] Add assertions --- .../admin/cluster/snapshots/get/GetSnapshotsResponse.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index 5c25e8dffc22b..2ad49c3c2f06c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -95,8 +95,10 @@ private GetSnapshotsResponse(List responses) { this.failedResponses = new HashMap<>(); for (Response response : responses) { if (response.snapshots != null) { + assert response.error == null; this.successfulResponses.put(response.repository, response.snapshots); } else { + assert response.snapshots == null; this.failedResponses.put(response.repository, response.error); } } From c00640fba25712c25cbe45cd96eaad41f224f738 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 23 May 2019 17:44:03 -0400 Subject: [PATCH 07/35] Use GetSnapshotsResponse.Response in GroupedActionListener --- .../snapshots/get/GetSnapshotsResponse.java | 29 ++++++++++-------- .../get/TransportGetSnapshotsAction.java | 30 ++++--------------- .../get/GetSnapshotsResponseTests.java | 10 +++---- 3 files changed, 26 insertions(+), 43 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index 2ad49c3c2f06c..6a40cc779beb5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -53,17 +54,13 @@ public class GetSnapshotsResponse extends ActionResponse implements ToXContentOb (p, c) -> Response.fromXContent(p), new ParseField("responses")); } - public GetSnapshotsResponse(Map> successfulResponses, Map failedResponses) { - this.successfulResponses = successfulResponses; - this.failedResponses = failedResponses; - } - private static class Response { - String repository; - List snapshots; - ElasticsearchException error; + public static class Response { + private String repository; + private List snapshots; + private ElasticsearchException error; - static final ConstructingObjectParser RESPONSE_PARSER = + private static final ConstructingObjectParser RESPONSE_PARSER = new ConstructingObjectParser<>(Response.class.getName(), true, (args) -> new Response((String) args[0], (List) args[1], (ElasticsearchException) args[2])); @@ -76,13 +73,21 @@ private static class Response { (p, c) -> ElasticsearchException.fromXContent(p), new ParseField("error")); } - Response(String repository, List snapshots, ElasticsearchException error) { + private Response(String repository, List snapshots, ElasticsearchException error) { this.repository = repository; this.snapshots = snapshots; this.error = error; } - static Response fromXContent(XContentParser parser) throws IOException { + public static Response snapshots(String repository, List snapshots) { + return new Response(repository, snapshots, null); + } + + public static Response error(String repository, ElasticsearchException error) { + return new Response(repository, null, error); + } + + private static Response fromXContent(XContentParser parser) throws IOException { return RESPONSE_PARSER.parse(parser, null); } } @@ -90,7 +95,7 @@ static Response fromXContent(XContentParser parser) throws IOException { private Map> successfulResponses = Collections.emptyMap(); private Map failedResponses = Collections.emptyMap(); - private GetSnapshotsResponse(List responses) { + public GetSnapshotsResponse(Collection responses) { this.successfulResponses = new HashMap<>(); this.failedResponses = new HashMap<>(); for (Response response : responses) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 384004016d83c..a8b4b275e2083 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -106,42 +106,22 @@ protected void masterOperation(final GetSnapshotsRequest request, final ClusterS private void getMultipleReposSnapshotInfo(List repos, String[] snapshots, boolean ignoreUnavailable, boolean verbose, ActionListener listener) { - GroupedActionListener, ElasticsearchException>>> groupedActionListener = + final GroupedActionListener groupedActionListener = new GroupedActionListener<>( ActionListener.map(listener, responses -> { assert repos.size() == responses.size(); - - Map> successfulResponses = new HashMap<>(); - Map failedResponses = new HashMap<>(); - - Iterator, ElasticsearchException>>> it = responses.iterator(); - - while (it.hasNext()) { - Tuple, ElasticsearchException>> response = it.next(); - String repo = response.v1(); - Tuple, ElasticsearchException> result = response.v2(); - if (result.v1() != null) { - assert result.v2() == null; - successfulResponses.put(repo, result.v1()); - } else { - assert result.v2() != null; - failedResponses.put(repo, result.v2()); - } - } - - return new GetSnapshotsResponse(successfulResponses, failedResponses); + return new GetSnapshotsResponse(responses); }), repos.size()); // run concurrently for all repos on GENERIC thread pool for (final RepositoryMetaData repo : repos) { threadPool.executor(ThreadPool.Names.GENERIC).execute( () -> { - // Unfortunately, there is no Either in Java, so we use Tuple with only one value set try { - groupedActionListener.onResponse(Tuple.tuple(repo.name(), - Tuple.tuple(getSingleRepoSnapshotInfo(repo.name(), snapshots, ignoreUnavailable, verbose), null))); + groupedActionListener.onResponse(GetSnapshotsResponse.Response.snapshots( + repo.name(), getSingleRepoSnapshotInfo(repo.name(), snapshots, ignoreUnavailable, verbose))); } catch (ElasticsearchException e) { - groupedActionListener.onResponse(Tuple.tuple(repo.name(), Tuple.tuple(null, e))); + groupedActionListener.onResponse(GetSnapshotsResponse.Response.error(repo.name(), e)); } }); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java index 5a70f7bf1ee96..2f2dd3fa0d1db 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java @@ -36,7 +36,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -86,22 +85,21 @@ private List createSnapshotInfos() { private GetSnapshotsResponse createTestInstance() { Set repositories = new HashSet<>(); - Map> successfulResponses = new HashMap<>(); - Map failedResponses = new HashMap<>(); + List responses = new ArrayList<>(); for (int i = 0; i < randomIntBetween(0, 5); i++) { String repository = randomValueOtherThanMany(r -> repositories.contains(r), () -> randomAlphaOfLength(10)); repositories.add(repository); - successfulResponses.put(repository, createSnapshotInfos()); + responses.add(GetSnapshotsResponse.Response.snapshots(repository, createSnapshotInfos())); } for (int i = 0; i < randomIntBetween(0, 5); i++) { String repository = randomValueOtherThanMany(r -> repositories.contains(r), () -> randomAlphaOfLength(10)); repositories.add(repository); - failedResponses.put(repository, new ElasticsearchException(randomAlphaOfLength(10))); + responses.add(GetSnapshotsResponse.Response.error(repository, new ElasticsearchException(randomAlphaOfLength(10)))); } - return new GetSnapshotsResponse(successfulResponses, failedResponses); + return new GetSnapshotsResponse(responses); } public void testSerialization() throws IOException { From 3963638ac7d0a5390c80943dcb5fe2031d4742ef Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 23 May 2019 17:48:49 -0400 Subject: [PATCH 08/35] Use unmodifiableMap in the constructor --- .../admin/cluster/snapshots/get/GetSnapshotsResponse.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index 6a40cc779beb5..81afefbd43e09 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -107,6 +107,8 @@ public GetSnapshotsResponse(Collection responses) { this.failedResponses.put(response.repository, response.error); } } + successfulResponses = Collections.unmodifiableMap(successfulResponses); + failedResponses = Collections.unmodifiableMap(failedResponses); } public GetSnapshotsResponse() { @@ -142,14 +144,14 @@ public Set getRepositories() { * Returns a map of repository name to the list of {@link SnapshotInfo} for each successful response. */ public Map> getSuccessfulResponses() { - return Map.copyOf(successfulResponses); + return successfulResponses; } /** * Returns a map of repository name to {@link ElasticsearchException} for each unsuccessful response. */ public Map getFailedResponses() { - return Map.copyOf(failedResponses); + return failedResponses; } /** From e8ec22c9314bea990e5d2e9e8654b85e91b6c91e Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 23 May 2019 18:06:51 -0400 Subject: [PATCH 09/35] Checkstyle --- .../cluster/snapshots/get/TransportGetSnapshotsAction.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index a8b4b275e2083..85ed506829e69 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -35,7 +35,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.repositories.IndexId; @@ -51,7 +50,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; From 0e956ae19586e8aab48cbcd8af1515128cd2c43a Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 24 May 2019 12:55:47 -0400 Subject: [PATCH 10/35] Moar checkstyle --- .../elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 3dc1a4571f360..5312844ce6c82 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -116,7 +116,6 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; From 71f6aa129b362eee53b7428481c467375c8037bf Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 24 May 2019 12:56:21 -0400 Subject: [PATCH 11/35] BWC --- .../admin/cluster/snapshots/get/GetSnapshotsRequest.java | 4 ++++ .../admin/cluster/snapshots/get/GetSnapshotsResponse.java | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java index 7c3f73ecf08bf..4bfd656700684 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java @@ -89,6 +89,10 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) { out.writeStringArray(repositories); } else { + if (repositories.length != 1) { + throw new IllegalArgumentException("Requesting snapshots from multiple repositories is not supported in versions prior " + + "to " + MULTIPLE_REPOSITORIES_SUPPORT_ADDED.toString()); + } out.writeString(repositories[0]); } out.writeStringArray(snapshots); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index 81afefbd43e09..89529591abdf7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -208,7 +208,8 @@ public void writeTo(StreamOutput out) throws IOException { } } else { if (successfulResponses.size() + failedResponses.size() != 1) { - throw new IllegalArgumentException("snapshots from more than one repository are requested"); + throw new IllegalArgumentException("Requesting snapshots from multiple repositories is not supported in versions prior " + + "to " + GetSnapshotsRequest.MULTIPLE_REPOSITORIES_SUPPORT_ADDED.toString()); } if (successfulResponses.size() == 1) { From adaa376e3dbc3213d9ea70a8b09a76e41abc0c63 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 24 May 2019 14:01:03 -0400 Subject: [PATCH 12/35] Fix indentation --- .../resources/rest-api-spec/test/cat.snapshots/10_basic.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml index 01af7c14454a1..f84233ff2c16f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml @@ -1,8 +1,8 @@ --- "Help": - skip: - version: " - 7.9.99" - reason: Repository field added in 8.0 + version: " - 7.9.99" + reason: Repository field added in 8.0 - do: cat.snapshots: From 85b2953ba5dc69173bda60c54365b6ceea370a10 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 24 May 2019 14:28:16 -0400 Subject: [PATCH 13/35] Add docs --- docs/reference/cat/snapshots.asciidoc | 14 +++++++++++++- .../migration/migrate_8_0/snapshots.asciidoc | 5 +++++ docs/reference/modules/snapshots.asciidoc | 8 ++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/reference/cat/snapshots.asciidoc b/docs/reference/cat/snapshots.asciidoc index 003514cc3a9d0..44e9b5c97e952 100644 --- a/docs/reference/cat/snapshots.asciidoc +++ b/docs/reference/cat/snapshots.asciidoc @@ -1,5 +1,5 @@ [[cat-snapshots]] -== cat snapshots +== Cat single repository snapshots The `snapshots` command shows all snapshots that belong to a specific repository. To find a list of available repositories to query, the command `/_cat/repositories` can be used. @@ -32,3 +32,15 @@ Each snapshot contains information about when it was started and stopped. Start and stop timestamps are available in two formats. The `HH:MM:SS` output is simply for quick human consumption. The epoch time retains more information, including date, and is machine sortable if the snapshot process spans days. + +== Cat multiple repositories snapshots +It is also possible to get the list of snapshots from multiple repositories. +Here are some examples: +[source,js] +-------------------------------------------------- +GET /_cat/snapshots/_all +GET /_cat/snapshots/repo1,repo2 +GET /_cat/snapshots/repo* +-------------------------------------------------- + +Please note that if one of the repositories fails during the request you will get an exception instead of the table. \ No newline at end of file diff --git a/docs/reference/migration/migrate_8_0/snapshots.asciidoc b/docs/reference/migration/migrate_8_0/snapshots.asciidoc index 791e5b28da057..c08ac76f88c90 100644 --- a/docs/reference/migration/migrate_8_0/snapshots.asciidoc +++ b/docs/reference/migration/migrate_8_0/snapshots.asciidoc @@ -9,6 +9,11 @@ // end::notable-breaking-changes[] +[float] +=== Get snapshots response format is changed +It's possible to get snapshots from multiple repositories in one go. The response format has changed +and now contains separate response for each repository. See <> for more information. + [float] ==== Deprecated node level compress setting removed diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index ec7916d5a3445..0dee2e6cfd6e6 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -465,6 +465,14 @@ that setting `verbose` to `false` will omit all other information about the snap such as status information, the number of snapshotted shards, etc. The default value of the `verbose` parameter is `true`. +It is also possible to retrieve snapshots for multiple repositories in one go, for example: +[source,sh] +----------------------------------- +GET /_snapshot/_all +GET /_snapshot/repo1,repo2 +GET /_snapshot/repo*/snap* +----------------------------------- + A currently running snapshot can be retrieved using the following command: [source,sh] From 5fa3d1693aab0f166a9187ef5b1907a9d8a5382a Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 24 May 2019 16:26:49 -0400 Subject: [PATCH 14/35] Fix docs --- docs/reference/cat/snapshots.asciidoc | 3 +++ docs/reference/modules/snapshots.asciidoc | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/reference/cat/snapshots.asciidoc b/docs/reference/cat/snapshots.asciidoc index 44e9b5c97e952..af8480f5c0994 100644 --- a/docs/reference/cat/snapshots.asciidoc +++ b/docs/reference/cat/snapshots.asciidoc @@ -36,11 +36,14 @@ The epoch time retains more information, including date, and is machine sortable == Cat multiple repositories snapshots It is also possible to get the list of snapshots from multiple repositories. Here are some examples: + [source,js] -------------------------------------------------- GET /_cat/snapshots/_all GET /_cat/snapshots/repo1,repo2 GET /_cat/snapshots/repo* -------------------------------------------------- +// CONSOLE +// TEST[skip:no repo2] Please note that if one of the repositories fails during the request you will get an exception instead of the table. \ No newline at end of file diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index 0dee2e6cfd6e6..6283b86d91e41 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -469,9 +469,11 @@ It is also possible to retrieve snapshots for multiple repositories in one go, f [source,sh] ----------------------------------- GET /_snapshot/_all -GET /_snapshot/repo1,repo2 -GET /_snapshot/repo*/snap* +GET /_snapshot/my_backup,my_fs_backup +GET /_snapshot/my*/snap* ----------------------------------- +// CONSOLE +// TEST[skip:no my_fs_backup] A currently running snapshot can be retrieved using the following command: From 3a8743f78c4576b2f4ebc3750522c7420b5a071e Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 24 May 2019 17:07:53 -0400 Subject: [PATCH 15/35] Fix FullClusterRestartIT --- .../org/elasticsearch/upgrades/FullClusterRestartIT.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 50eee32810adc..3de5f8007df6e 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -1053,9 +1053,11 @@ private void checkSnapshot(final String snapshotName, final int count, final Ver // Check the snapshot metadata, especially the version Request listSnapshotRequest = new Request("GET", "/_snapshot/repo/" + snapshotName); Map listSnapshotResponse = entityAsMap(client().performRequest(listSnapshotRequest)); - assertEquals(singletonList(snapshotName), XContentMapValues.extractValue("snapshots.snapshot", listSnapshotResponse)); - assertEquals(singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", listSnapshotResponse)); - assertEquals(singletonList(tookOnVersion.toString()), XContentMapValues.extractValue("snapshots.version", listSnapshotResponse)); + assertEquals(singletonList(snapshotName), XContentMapValues.extractValue("responses[0].snapshots.snapshot", + listSnapshotResponse)); + assertEquals(singletonList("SUCCESS"), XContentMapValues.extractValue("responses[0].snapshots.state", listSnapshotResponse)); + assertEquals(singletonList(tookOnVersion.toString()), XContentMapValues.extractValue("responses[0].snapshots.version", + listSnapshotResponse)); // Remove the routing setting and template so we can test restoring them. Request clearRoutingFromSettings = new Request("PUT", "/_cluster/settings"); From ec8bce76ef1b8c0383ae9c7b4f2654739fe06a35 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 28 May 2019 13:28:05 -0400 Subject: [PATCH 16/35] Fix tests --- .../upgrades/FullClusterRestartIT.java | 19 ++++++---- .../test/cat.snapshots/10_basic.yml | 4 +- .../test/snapshot.get/10_basic.yml | 38 ++++++++++--------- .../ESBlobStoreRepositoryIntegTestCase.java | 2 +- .../indexlifecycle/CCRIndexLifecycleIT.java | 8 ++-- 5 files changed, 39 insertions(+), 32 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 3de5f8007df6e..b195c020aa7ca 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -1049,21 +1049,26 @@ private void assertClosedIndex(final String index, final boolean checkRoutingTab } } + @SuppressWarnings("unchecked") private void checkSnapshot(final String snapshotName, final int count, final Version tookOnVersion) throws IOException { // Check the snapshot metadata, especially the version Request listSnapshotRequest = new Request("GET", "/_snapshot/repo/" + snapshotName); - Map listSnapshotResponse = entityAsMap(client().performRequest(listSnapshotRequest)); - assertEquals(singletonList(snapshotName), XContentMapValues.extractValue("responses[0].snapshots.snapshot", - listSnapshotResponse)); - assertEquals(singletonList("SUCCESS"), XContentMapValues.extractValue("responses[0].snapshots.state", listSnapshotResponse)); - assertEquals(singletonList(tookOnVersion.toString()), XContentMapValues.extractValue("responses[0].snapshots.version", - listSnapshotResponse)); + Map responseMap = entityAsMap(client().performRequest(listSnapshotRequest)); + Map snapResponse; + if (responseMap.get("responses") != null) { + snapResponse = (Map) ((List) responseMap.get("responses")).get(0); + } else { + snapResponse = responseMap; + } + + assertEquals(singletonList(snapshotName), XContentMapValues.extractValue("snapshots.snapshot", snapResponse)); + assertEquals(singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", snapResponse)); + assertEquals(singletonList(tookOnVersion.toString()), XContentMapValues.extractValue("snapshots.version", snapResponse)); // Remove the routing setting and template so we can test restoring them. Request clearRoutingFromSettings = new Request("PUT", "/_cluster/settings"); clearRoutingFromSettings.setJsonEntity("{\"persistent\":{\"cluster.routing.allocation.exclude.test_attr\": null}}"); client().performRequest(clearRoutingFromSettings); - client().performRequest(new Request("DELETE", "/_template/test_template")); // Restore diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml index f84233ff2c16f..4f425f4817638 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml @@ -82,6 +82,6 @@ - match: $body: | - /^ snap1\s+ test_cat_snapshots_1\+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n - snap2\s+ test_cat_snapshots_1\+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n + /^ snap1\s+ test_cat_snapshots_1\s+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n + snap2\s+ test_cat_snapshots_1\s+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml index e2b7279f1cd68..fde9d1f98daaf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml @@ -31,8 +31,8 @@ setup: repository: test_repo_get_1 snapshot: test_snapshot - - is_true: snapshots - - is_true: snapshots.0.failures + - is_true: responses.0.snapshots + - is_true: responses.0.snapshots.0.failures - do: snapshot.delete: @@ -43,11 +43,13 @@ setup: "Get missing snapshot info throws an exception": - do: - catch: /snapshot_missing_exception.+ is missing/ snapshot.get: repository: test_repo_get_1 snapshot: test_nonexistent_snapshot + - is_true: responses.0.error + - match: { responses.0.error.type: snapshot_missing_exception } + --- "Get missing snapshot info succeeds when ignore_unavailable is true": @@ -57,7 +59,7 @@ setup: snapshot: test_nonexistent_snapshot ignore_unavailable: true - - is_true: snapshots + - is_true: responses.0.snapshots --- "Get snapshot info when verbose is false": @@ -81,12 +83,12 @@ setup: snapshot: test_snapshot verbose: false - - is_true: snapshots - - match: { snapshots.0.snapshot: test_snapshot } - - match: { snapshots.0.state: SUCCESS } - - is_false: snapshots.0.failures - - is_false: snapshots.0.shards - - is_false: snapshots.0.version + - is_true: responses.0.snapshots + - match: { responses.0.snapshots.0.snapshot: test_snapshot } + - match: { responses.0.snapshots.0.state: SUCCESS } + - is_false: responses.0.snapshots.0.failures + - is_false: responses.0.snapshots.0.shards + - is_false: responses.0.snapshots.0.version - do: snapshot.delete: @@ -117,10 +119,10 @@ setup: repository: test_repo_get_1 snapshot: test_snapshot_with_include_global_state - - is_true: snapshots - - match: { snapshots.0.snapshot: test_snapshot_with_include_global_state } - - match: { snapshots.0.state: SUCCESS } - - match: { snapshots.0.include_global_state: true } + - is_true: responses.0.snapshots + - match: { responses.0.snapshots.0.snapshot: test_snapshot_with_include_global_state } + - match: { responses.0.snapshots.0.state: SUCCESS } + - match: { responses.0.snapshots.0.include_global_state: true } - do: snapshot.delete: @@ -140,10 +142,10 @@ setup: repository: test_repo_get_1 snapshot: test_snapshot_without_include_global_state - - is_true: snapshots - - match: { snapshots.0.snapshot: test_snapshot_without_include_global_state } - - match: { snapshots.0.state: SUCCESS } - - match: { snapshots.0.include_global_state: false } + - is_true: responses.0.snapshots + - match: { responses.0.snapshots.0.snapshot: test_snapshot_without_include_global_state } + - match: { responses.0.snapshots.0.state: SUCCESS } + - match: { responses.0.snapshots.0.include_global_state: false } - do: snapshot.delete: diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java index 8187a46fa7425..947a2c9887c10 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java @@ -148,7 +148,7 @@ public void testSnapshotAndRestore() throws Exception { assertAcked(client().admin().cluster().prepareDeleteSnapshot(repoName, snapshotName).get()); expectThrows(SnapshotMissingException.class, () -> - client().admin().cluster().prepareGetSnapshots(repoName).setSnapshots(snapshotName).get()); + client().admin().cluster().prepareGetSnapshots(repoName).setSnapshots(snapshotName).get().getSnapshots(repoName)); expectThrows(SnapshotMissingException.class, () -> client().admin().cluster().prepareDeleteSnapshot(repoName, snapshotName).get()); diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java index b185a425934eb..de91e40a65d72 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java @@ -171,9 +171,6 @@ public void testCCRUnfollowDuringSnapshot() throws Exception { // assert that snapshot succeeded assertThat(getSnapshotState("snapshot"), equalTo("SUCCESS")); assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot"))); - ResponseException e = expectThrows(ResponseException.class, - () -> client().performRequest(new Request("GET", "/_snapshot/repo/snapshot"))); - assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); } } else { fail("unexpected target cluster [" + targetCluster + "]"); @@ -706,13 +703,16 @@ public static void updatePolicy(String indexName, String policy) throws IOExcept assertOK(client().performRequest(changePolicyRequest)); } + @SuppressWarnings("unchecked") private String getSnapshotState(String snapshot) throws IOException { Response response = client().performRequest(new Request("GET", "/_snapshot/repo/" + snapshot)); Map responseMap; try (InputStream is = response.getEntity().getContent()) { responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true); } - @SuppressWarnings("unchecked") Map snapResponse = ((List>) responseMap.get("snapshots")).get(0); + + Map repoResponse = ((List>) responseMap.get("responses")).get(0); + Map snapResponse = ((List>) repoResponse.get("snapshots")).get(0); assertThat(snapResponse.get("snapshot"), equalTo(snapshot)); return (String) snapResponse.get("state"); } From 12a563d0657e169497bc9ce219098483f5115851 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 28 May 2019 13:50:05 -0400 Subject: [PATCH 17/35] Checkstyle --- .../elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java index de91e40a65d72..757d1dc15907b 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; From 62a7acc31d2ab70aca2fdb3eeab238797494f161 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 28 May 2019 14:54:51 -0400 Subject: [PATCH 18/35] Moar fixes --- .../test/repository_url/10_basic.yml | 17 +++++++++-------- .../TimeSeriesLifecycleActionsIT.java | 5 ++++- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml index b932f0d53caad..c43fe1659aa90 100644 --- a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml +++ b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml @@ -126,9 +126,9 @@ teardown: repository: repository-url snapshot: snapshot-one,snapshot-two - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } + - is_true: responses.0.snapshots + - match: { responses.0.snapshots.0.state : SUCCESS } + - match: { responses.0.snapshots.1.state : SUCCESS } # Delete the index - do: @@ -188,9 +188,9 @@ teardown: repository: repository-file snapshot: snapshot-one,snapshot-two - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } + - is_true: responses.0.snapshots + - match: { responses.0.snapshots.0.state : SUCCESS } + - match: { responses.0.snapshots.1.state : SUCCESS } # Delete the index - do: @@ -236,13 +236,14 @@ teardown: --- "Get a non existing snapshot": - - do: - catch: /snapshot_missing_exception/ snapshot.get: repository: repository-url snapshot: missing + - is_true: responses.0.error + - match: { responses.0.error.type: snapshot_missing_exception } + --- "Delete a non existing snapshot": diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index b6b317e0c67ef..9df576eb220a9 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -977,13 +977,16 @@ private void indexDocument() throws IOException { logger.info(response.getStatusLine()); } + @SuppressWarnings("unchecked") private String getSnapshotState(String snapshot) throws IOException { Response response = client().performRequest(new Request("GET", "/_snapshot/repo/" + snapshot)); Map responseMap; try (InputStream is = response.getEntity().getContent()) { responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true); } - Map snapResponse = ((List>) responseMap.get("snapshots")).get(0); + + Map repoResponse = ((List>) responseMap.get("responses")).get(0); + Map snapResponse = ((List>) repoResponse.get("snapshots")).get(0); assertThat(snapResponse.get("snapshot"), equalTo(snapshot)); return (String) snapResponse.get("state"); } From 94d3a5bfde6c592853cd41e6ad006daf69ce2c10 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 28 May 2019 15:18:21 -0400 Subject: [PATCH 19/35] Fix --- .../admin/cluster/snapshots/get/GetSnapshotsResponse.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index 89529591abdf7..0fb2667a78435 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -220,7 +220,9 @@ public void writeTo(StreamOutput out) throws IOException { } } - throw failedResponses.values().iterator().next(); + if (failedResponses.isEmpty() == false) { + throw failedResponses.values().iterator().next(); + } } } From cf0bbda3a5ba0aac57fc3bc93e9e3e402130a4ba Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 28 May 2019 15:27:42 -0400 Subject: [PATCH 20/35] Fix merge --- .../repositories/AbstractThirdPartyRepositoryTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java index 90c399a5af6c7..1de3a8ff2f688 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java @@ -76,7 +76,7 @@ public void testCreateSnapshot() { .prepareGetSnapshots("test-repo") .setSnapshots(snapshotName) .get() - .getSnapshots() + .getSnapshots("test-repo") .get(0) .state(), equalTo(SnapshotState.SUCCESS)); From ff4c89b48b6bd888275ef24cc1e0a4a2e101b39a Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 31 May 2019 15:07:17 +0200 Subject: [PATCH 21/35] More fixes --- .../test/repository_url/10_basic.yml | 10 ++++++++++ .../test/repository_azure/10_repository.yml | 16 ++++++++++++---- .../test/repository_gcs/10_repository.yml | 16 ++++++++++++---- .../rest-api-spec/test/snapshot.get/10_basic.yml | 16 ++++++++++++++++ .../snapshots/get/GetSnapshotsResponse.java | 4 +++- .../elasticsearch/test/rest/ESRestTestCase.java | 13 ++++++++++--- .../TimeSeriesLifecycleActionsIT.java | 9 --------- 7 files changed, 63 insertions(+), 21 deletions(-) diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml index c43fe1659aa90..ba4d101a6683a 100644 --- a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml +++ b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml @@ -112,6 +112,9 @@ teardown: --- "Restore with repository-url using http://": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" # Ensure that the URL repository is registered - do: @@ -174,6 +177,9 @@ teardown: --- "Restore with repository-url using file://": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" # Ensure that the URL repository is registered - do: @@ -236,6 +242,10 @@ teardown: --- "Get a non existing snapshot": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" + - do: snapshot.get: repository: repository-url diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml index 92866190959e6..fade1f9f1e67d 100644 --- a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml +++ b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml @@ -15,6 +15,9 @@ setup: --- "Snapshot/Restore with repository-azure": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" # Get repository - do: @@ -121,9 +124,9 @@ setup: repository: repository snapshot: snapshot-one,snapshot-two - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } + - is_true: responses.0.snapshots + - match: { responses.0.snapshots.0.state: SUCCESS } + - match: { responses.0.snapshots.1.state: SUCCESS } # Delete the index - do: @@ -203,13 +206,18 @@ setup: --- "Get a non existing snapshot": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" - do: - catch: /snapshot_missing_exception/ snapshot.get: repository: repository snapshot: missing + - is_true: responses.0.error + - match: { responses.0.error.type: snapshot_missing_exception } + --- "Delete a non existing snapshot": diff --git a/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml b/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml index ac649229001db..553b6a3e14e50 100644 --- a/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml +++ b/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml @@ -28,6 +28,9 @@ setup: --- "Snapshot/Restore with repository-gcs": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" # Get repository - do: @@ -134,9 +137,9 @@ setup: repository: repository snapshot: snapshot-one,snapshot-two - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } + - is_true: responses.0.snapshots + - match: { responses.0.snapshots.0.state : SUCCESS } + - match: { responses.0.snapshots.1.state : SUCCESS } # Delete the index - do: @@ -213,13 +216,18 @@ setup: --- "Get a non existing snapshot": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" - do: - catch: /snapshot_missing_exception/ snapshot.get: repository: repository snapshot: missing + - is_true: responses.0.error + - match: { responses.0.error.type: snapshot_missing_exception } + --- "Delete a non existing snapshot": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml index fde9d1f98daaf..a4a4f23bb60ba 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml @@ -11,6 +11,9 @@ setup: --- "Get snapshot info": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" - do: indices.create: @@ -41,6 +44,9 @@ setup: --- "Get missing snapshot info throws an exception": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" - do: snapshot.get: @@ -52,6 +58,9 @@ setup: --- "Get missing snapshot info succeeds when ignore_unavailable is true": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" - do: snapshot.get: @@ -63,6 +72,10 @@ setup: --- "Get snapshot info when verbose is false": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" + - do: indices.create: index: test_index @@ -97,6 +110,9 @@ setup: --- "Get snapshot info contains include_global_state": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" - do: indices.create: diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index 0fb2667a78435..7ca6806ce58b0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -249,13 +249,15 @@ public void readFrom(StreamInput in) throws IOException { ElasticsearchException error = in.readException(); failedResponses.put(repository, error); } + successfulResponses = Collections.unmodifiableMap(successfulResponses); + failedResponses = Collections.unmodifiableMap(failedResponses); } else { int size = in.readVInt(); List snapshots = new ArrayList<>(size); for (int i = 0; i < size; i++) { snapshots.add(new SnapshotInfo(in)); } - successfulResponses.put("unknown", snapshots); + successfulResponses = Collections.singletonMap("unknown", snapshots); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index e7fc73fe985b9..90fc04400c476 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -540,9 +540,16 @@ private void wipeCluster() throws Exception { // All other repo types we really don't have a chance of being able to iterate properly, sadly. Request listRequest = new Request("GET", "/_snapshot/" + repoName + "/_all"); listRequest.addParameter("ignore_unavailable", "true"); - List snapshots = - (List)((Map)((List) entityAsMap(adminClient.performRequest(listRequest)).get("responses")).get(0)) - .get("snapshots"); + + Map response = entityAsMap(adminClient.performRequest(listRequest)); + Map oneRepoResponse; + if (response.containsKey("responses")) { + oneRepoResponse = ((Map)((List) response.get("responses")).get(0)); + } else { + oneRepoResponse = response; + } + + List snapshots = (List) oneRepoResponse.get("snapshots"); for (Object snapshot : snapshots) { Map snapshotInfo = (Map) snapshot; String name = (String) snapshotInfo.get("snapshot"); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index 9df576eb220a9..8d1254a6abfc5 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -393,9 +393,6 @@ public void testDeleteDuringSnapshot() throws Exception { // assert that snapshot is still in progress and clean up assertThat(getSnapshotState("snapshot"), equalTo("SUCCESS")); assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot"))); - ResponseException e = expectThrows(ResponseException.class, - () -> client().performRequest(new Request("GET", "/_snapshot/repo/snapshot"))); - assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); } public void testReadOnly() throws Exception { @@ -533,9 +530,6 @@ public void testShrinkDuringSnapshot() throws Exception { // assert that snapshot succeeded assertThat(getSnapshotState("snapshot"), equalTo("SUCCESS")); assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot"))); - ResponseException e = expectThrows(ResponseException.class, - () -> client().performRequest(new Request("GET", "/_snapshot/repo/snapshot"))); - assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); } public void testFreezeAction() throws Exception { @@ -591,9 +585,6 @@ public void testFreezeDuringSnapshot() throws Exception { // assert that snapshot is still in progress and clean up assertThat(getSnapshotState("snapshot"), equalTo("SUCCESS")); assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot"))); - ResponseException e = expectThrows(ResponseException.class, - () -> client().performRequest(new Request("GET", "/_snapshot/repo/snapshot"))); - assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); } public void testSetPriority() throws Exception { From 16c69ac6e55c5dae65ab4634f1269979e7186816 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 31 May 2019 15:50:00 +0200 Subject: [PATCH 22/35] Fix yml --- .../rest-api-spec/test/repository_url/10_basic.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml index ba4d101a6683a..e76669dc75e01 100644 --- a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml +++ b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml @@ -178,8 +178,8 @@ teardown: --- "Restore with repository-url using file://": - skip: - version: " - 7.9.99" - reason: "8.0 changes get snapshots response format" + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" # Ensure that the URL repository is registered - do: @@ -243,8 +243,8 @@ teardown: --- "Get a non existing snapshot": - skip: - version: " - 7.9.99" - reason: "8.0 changes get snapshots response format" + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" - do: snapshot.get: From 265f18e7c16b24bcefa0cbb26605c6f349b233b4 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 31 May 2019 17:01:32 +0200 Subject: [PATCH 23/35] s3 yml fixes --- .../20_repository_permanent_credentials.yml | 17 ++++++++++++----- .../30_repository_temporary_credentials.yml | 16 ++++++++++++---- .../40_repository_ec2_credentials.yml | 16 ++++++++++++---- .../50_repository_ecs_credentials.yml | 16 ++++++++++++---- 4 files changed, 48 insertions(+), 17 deletions(-) diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml index e6c94f8c408d9..d319bf8984a97 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml @@ -31,7 +31,6 @@ setup: --- "Try to create repository with broken endpoint override and named client": - # Register repository with broken endpoint setting - do: catch: /repository_verification_exception/ @@ -108,6 +107,9 @@ setup: --- "Snapshot and Restore with repository-s3 using permanent credentials": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" # Get repository - do: @@ -215,9 +217,9 @@ setup: repository: repository_permanent snapshot: snapshot-one,snapshot-two - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } + - is_true: responses.0.snapshots + - match: { responses.0.snapshots.0.state : SUCCESS } + - match: { responses.0.snapshots.1.state : SUCCESS } # Delete the index - do: @@ -322,13 +324,18 @@ setup: --- "Get a non existing snapshot": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" - do: - catch: /snapshot_missing_exception/ snapshot.get: repository: repository_permanent snapshot: missing + - is_true: responses.0.error + - match: { responses.0.error.type: snapshot_missing_exception } + --- "Delete a non existing snapshot": diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml index d5bdcd9c4f203..3ad6c3959634b 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml @@ -18,6 +18,9 @@ setup: --- "Snapshot and Restore with repository-s3 using temporary credentials": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" # Get repository - do: @@ -122,9 +125,9 @@ setup: repository: repository_temporary snapshot: snapshot-one,snapshot-two - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } + - is_true: responses.0.snapshots + - match: { responses.0.snapshots.0.state : SUCCESS } + - match: { responses.0.snapshots.1.state : SUCCESS } # Delete the index - do: @@ -229,13 +232,18 @@ setup: --- "Get a non existing snapshot": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" - do: - catch: /snapshot_missing_exception/ snapshot.get: repository: repository_temporary snapshot: missing + - is_true: responses.0.error + - match: { responses.0.error.type: snapshot_missing_exception } + --- "Delete a non existing snapshot": diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml index 829ae6197659c..fa1d3fc10fb13 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml @@ -18,6 +18,9 @@ setup: --- "Snapshot and Restore with repository-s3 using ec2 credentials": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" # Get repository - do: @@ -122,9 +125,9 @@ setup: repository: repository_ec2 snapshot: snapshot-one,snapshot-two - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } + - is_true: responses.0.snapshots + - match: { responses.0.snapshots.0.state : SUCCESS } + - match: { responses.0.snapshots.1.state : SUCCESS } # Delete the index - do: @@ -229,13 +232,18 @@ setup: --- "Get a non existing snapshot": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" - do: - catch: /snapshot_missing_exception/ snapshot.get: repository: repository_ec2 snapshot: missing + - is_true: responses.0.error + - match: { responses.0.error.type: snapshot_missing_exception } + --- "Delete a non existing snapshot": diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml index c59d3a32badc7..99736fb25ff24 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml @@ -18,6 +18,9 @@ setup: --- "Snapshot and Restore with repository-s3 using ecs credentials": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" # Get repository - do: @@ -122,9 +125,9 @@ setup: repository: repository_ecs snapshot: snapshot-one,snapshot-two - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } + - is_true: responses.0.snapshots + - match: { responses.0.snapshots.0.state : SUCCESS } + - match: { responses.0.snapshots.1.state : SUCCESS } # Delete the index - do: @@ -229,13 +232,18 @@ setup: --- "Get a non existing snapshot": + - skip: + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" - do: - catch: /snapshot_missing_exception/ snapshot.get: repository: repository_ecs snapshot: missing + - is_true: responses.0.error + - match: { responses.0.error.type: snapshot_missing_exception } + --- "Delete a non existing snapshot": From be898f4f18a5b7c3c5f872abe2271e0900c0da01 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 3 Jun 2019 15:38:08 +0200 Subject: [PATCH 24/35] Revert reformatting --- .../cluster/snapshots/get/TransportGetSnapshotsAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 85ed506829e69..9054e970ada01 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -66,7 +66,7 @@ public TransportGetSnapshotsAction(TransportService transportService, ClusterSer ThreadPool threadPool, SnapshotsService snapshotsService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { super(GetSnapshotsAction.NAME, transportService, clusterService, threadPool, actionFilters, - GetSnapshotsRequest::new, indexNameExpressionResolver); + GetSnapshotsRequest::new, indexNameExpressionResolver); this.snapshotsService = snapshotsService; } From 33cad5e4d79c35436227a49fcc3628df877070c3 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 3 Jun 2019 15:42:14 +0200 Subject: [PATCH 25/35] Use ActionRunnable --- .../get/TransportGetSnapshotsAction.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 9054e970ada01..5f7499b554674 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesAction; import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; @@ -113,15 +114,17 @@ private void getMultipleReposSnapshotInfo(List repos, String // run concurrently for all repos on GENERIC thread pool for (final RepositoryMetaData repo : repos) { - threadPool.executor(ThreadPool.Names.GENERIC).execute( - () -> { - try { - groupedActionListener.onResponse(GetSnapshotsResponse.Response.snapshots( - repo.name(), getSingleRepoSnapshotInfo(repo.name(), snapshots, ignoreUnavailable, verbose))); - } catch (ElasticsearchException e) { - groupedActionListener.onResponse(GetSnapshotsResponse.Response.error(repo.name(), e)); - } - }); + threadPool.executor(ThreadPool.Names.GENERIC).execute(new ActionRunnable<>(groupedActionListener) { + @Override + protected void doRun() { + try { + groupedActionListener.onResponse(GetSnapshotsResponse.Response.snapshots( + repo.name(), getSingleRepoSnapshotInfo(repo.name(), snapshots, ignoreUnavailable, verbose))); + } catch (ElasticsearchException e) { + groupedActionListener.onResponse(GetSnapshotsResponse.Response.error(repo.name(), e)); + } + } + }); } } From ca011e388ecab53e3a11c16009c9b8212919f695 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 3 Jun 2019 15:45:03 +0200 Subject: [PATCH 26/35] buildSimpleSnapshotInfos --- .../cluster/snapshots/get/TransportGetSnapshotsAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 5f7499b554674..7ae8aca4b278a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -183,7 +183,7 @@ private List getSingleRepoSnapshotInfo(String repo, String[] snaps } else { if (repositoryData != null) { // want non-current snapshots as well, which are found in the repository data - snapshotInfos = buildSimpleSnapshotInfos(repo, toResolve, repositoryData, currentSnapshots); + snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots); } else { // only want current snapshots snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList()); @@ -202,7 +202,7 @@ private boolean isCurrentSnapshotsOnly(String[] snapshots) { return (snapshots.length == 1 && GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshots[0])); } - private List buildSimpleSnapshotInfos(final String repository, final Set toResolve, + private static List buildSimpleSnapshotInfos(final Set toResolve, final RepositoryData repositoryData, final List currentSnapshots) { List snapshotInfos = new ArrayList<>(); From 84430b217f3d1565edfb4cc75ad6cfe84c32355b Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 14 Jun 2019 11:35:11 +0300 Subject: [PATCH 27/35] Fix typo lost during force push --- docs/reference/modules/snapshots.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index ff8989cfa3ab7..a2fcd39496092 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -472,7 +472,7 @@ that setting `verbose` to `false` will omit all other information about the snap such as status information, the number of snapshotted shards, etc. The default value of the `verbose` parameter is `true`. -It is also possible to retrieve snapshots for multiple repositories in one go, for example: +It is also possible to retrieve snapshots from multiple repositories in one go, for example: [source,sh] ----------------------------------- GET /_snapshot/_all From 65825086ca8366f26f8cbe64da371d2273ec3694 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 14 Jun 2019 11:48:24 +0300 Subject: [PATCH 28/35] Change cat shapshots docs --- docs/reference/cat/snapshots.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/cat/snapshots.asciidoc b/docs/reference/cat/snapshots.asciidoc index 9f2c613daf835..37d57292e0488 100644 --- a/docs/reference/cat/snapshots.asciidoc +++ b/docs/reference/cat/snapshots.asciidoc @@ -1,7 +1,8 @@ [[cat-snapshots]] -== Cat single repository snapshots +== cat snapshots -The `snapshots` command shows all snapshots that belong to a specific repository. +The `snapshots` command shows all snapshots that belong to a specific repository +or multiple repositories. To find a list of available repositories to query, the command `/_cat/repositories` can be used. Querying the snapshots of a repository named `repo1` then looks as follows. @@ -33,7 +34,6 @@ Start and stop timestamps are available in two formats. The `HH:MM:SS` output is simply for quick human consumption. The epoch time retains more information, including date, and is machine sortable if the snapshot process spans days. -== Cat multiple repositories snapshots It is also possible to get the list of snapshots from multiple repositories. Here are some examples: From 35a171fd393f167d813817128950573843355e29 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 14 Jun 2019 12:20:07 +0300 Subject: [PATCH 29/35] \n --- .../main/resources/rest-api-spec/test/snapshot.get/10_basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml index c627542af4cb2..bf25b0c1864a9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml @@ -205,4 +205,4 @@ setup: - do: snapshot.delete: repository: test_repo_get_1 - snapshot: test_snapshot_with_metadata \ No newline at end of file + snapshot: test_snapshot_with_metadata From dc64cea8a84b9c55c95a81876fbd0af4216351af Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 14 Jun 2019 14:39:55 +0300 Subject: [PATCH 30/35] Fix "Get snapshot info with metadata" --- .../rest-api-spec/test/snapshot.get/10_basic.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml index bf25b0c1864a9..219c5a70f331e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml @@ -172,8 +172,8 @@ setup: --- "Get snapshot info with metadata": - skip: - version: " - 7.2.99" - reason: "metadata field was added in 7.3" + version: " - 7.9.99" + reason: "8.0 changes get snapshots response format" - do: indices.create: @@ -196,11 +196,11 @@ setup: repository: test_repo_get_1 snapshot: test_snapshot_with_metadata - - is_true: snapshots - - match: { snapshots.0.snapshot: test_snapshot_with_metadata } - - match: { snapshots.0.state: SUCCESS } - - match: { snapshots.0.metadata.taken_by: test } - - match: { snapshots.0.metadata.foo.bar: baz } + - is_true: responses.0.snapshots + - match: { responses.0.snapshots.0.snapshot: test_snapshot_with_metadata } + - match: { responses.0.snapshots.0.state: SUCCESS } + - match: { responses.0.snapshots.0.metadata.taken_by: test } + - match: { responses.0.snapshots.0.metadata.foo.bar: baz } - do: snapshot.delete: From 682d583121441f8ee5d1982659d4e1c0b7a4c2ec Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 17 Jun 2019 14:58:27 +0300 Subject: [PATCH 31/35] readFrom should throw UnsupportedOpExc --- .../cluster/repositories/get/GetRepositoriesResponse.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java index add8e13b26ac6..8063b348c2576 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java @@ -50,7 +50,7 @@ public class GetRepositoriesResponse extends ActionResponse implements ToXConten } public GetRepositoriesResponse(StreamInput in) throws IOException { - readFrom(in); + repositories = new RepositoriesMetaData(in); } /** @@ -65,7 +65,7 @@ public List repositories() { @Override public void readFrom(StreamInput in) throws IOException { - repositories = new RepositoriesMetaData(in); + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); } @Override From f19ff8f3c6372928d71c50cf0f4f9858dc3c93c4 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 17 Jun 2019 15:02:12 +0300 Subject: [PATCH 32/35] Add exceptions as cause in cat API --- .../rest/action/cat/RestSnapshotAction.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java index ba6cb42da257a..4b679cbf475eb 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java @@ -109,10 +109,19 @@ private Table buildTable(RestRequest req, GetSnapshotsResponse getSnapshotsRespo Table table = getTableWithHeader(req); if (getSnapshotsResponse.isFailed()) { + ElasticsearchException causes = null; + + for (ElasticsearchException e : getSnapshotsResponse.getFailedResponses().values()) { + if (causes == null) { + causes = e; + } else { + causes.addSuppressed(e); + } + } throw new ElasticsearchException( "Repositories [" + Strings.collectionToCommaDelimitedString(getSnapshotsResponse.getFailedResponses().keySet()) + - "] failed to retrieve snapshots"); + "] failed to retrieve snapshots", causes); } for (Map.Entry> response : getSnapshotsResponse.getSuccessfulResponses().entrySet()) { From e2f2e9a28192302f06a7c433a8afb5cd787288d2 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 17 Jun 2019 15:03:48 +0300 Subject: [PATCH 33/35] Make repositories final --- .../admin/cluster/repositories/get/GetRepositoriesResponse.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java index 8063b348c2576..102d59a217755 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java @@ -39,7 +39,7 @@ */ public class GetRepositoriesResponse extends ActionResponse implements ToXContentObject { - private RepositoriesMetaData repositories; + private final RepositoriesMetaData repositories; GetRepositoriesResponse() { repositories = new RepositoriesMetaData(Collections.emptyList()); From e44103294d74d4765d7de5ba91cf13a38417fb50 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 17 Jun 2019 16:42:08 +0300 Subject: [PATCH 34/35] Fix GetRepositoriesResponse streamable and get rid of rid of readFrom method in GetSnapshotsResponse --- .../get/GetRepositoriesAction.java | 8 +- .../get/GetRepositoriesResponse.java | 5 -- .../get/TransportGetRepositoriesAction.java | 9 +- .../snapshots/get/GetSnapshotsAction.java | 8 +- .../snapshots/get/GetSnapshotsResponse.java | 85 ++++++++++--------- .../get/TransportGetSnapshotsAction.java | 9 +- .../get/GetSnapshotsResponseTests.java | 4 +- 7 files changed, 76 insertions(+), 52 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesAction.java index d89e466461d9b..eac4d971b60a1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.cluster.repositories.get; import org.elasticsearch.action.Action; +import org.elasticsearch.common.io.stream.Writeable; /** * Get repositories action @@ -35,7 +36,12 @@ private GetRepositoriesAction() { @Override public GetRepositoriesResponse newResponse() { - return new GetRepositoriesResponse(); + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public Writeable.Reader getResponseReader() { + return GetRepositoriesResponse::new; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java index 102d59a217755..4fb9cbaebe941 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; -import java.util.Collections; import java.util.List; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -41,10 +40,6 @@ public class GetRepositoriesResponse extends ActionResponse implements ToXConten private final RepositoriesMetaData repositories; - GetRepositoriesResponse() { - repositories = new RepositoriesMetaData(Collections.emptyList()); - } - GetRepositoriesResponse(RepositoriesMetaData repositories) { this.repositories = repositories; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java index 4b3ee1cd9251a..c626a17a13ead 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java @@ -31,11 +31,13 @@ import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashSet; @@ -62,7 +64,12 @@ protected String executor() { @Override protected GetRepositoriesResponse newResponse() { - return new GetRepositoriesResponse(); + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + protected GetRepositoriesResponse read(StreamInput in) throws IOException { + return new GetRepositoriesResponse(in); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsAction.java index b5015ff5c23b0..b77130f1875be 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.cluster.snapshots.get; import org.elasticsearch.action.Action; +import org.elasticsearch.common.io.stream.Writeable; /** * Get snapshots action @@ -35,7 +36,12 @@ private GetSnapshotsAction() { @Override public GetSnapshotsResponse newResponse() { - return new GetSnapshotsResponse(); + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public Writeable.Reader getResponseReader() { + return GetSnapshotsResponse::new; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index 014cca2514995..1ecbd222c6d84 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -55,6 +55,40 @@ public class GetSnapshotsResponse extends ActionResponse implements ToXContentOb (p, c) -> Response.fromXContent(p), new ParseField("responses")); } + public GetSnapshotsResponse(StreamInput in) throws IOException { + if (in.getVersion().onOrAfter(GetSnapshotsRequest.MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) { + int successfulSize = in.readVInt(); + Map> successfulResponses = new HashMap<>(successfulSize); + for (int i = 0; i < successfulSize; i++) { + String repository = in.readString(); + int size = in.readVInt(); + List snapshotInfos = new ArrayList<>(size); + for (int j = 0; j < size; j++) { + snapshotInfos.add(new SnapshotInfo(in)); + } + successfulResponses.put(repository, snapshotInfos); + } + + int failedSize = in.readVInt(); + Map failedResponses = new HashMap<>(failedSize); + for (int i = 0; i < failedSize; i++) { + String repository = in.readString(); + ElasticsearchException error = in.readException(); + failedResponses.put(repository, error); + } + this.successfulResponses = Collections.unmodifiableMap(successfulResponses); + this.failedResponses = Collections.unmodifiableMap(failedResponses); + } else { + int size = in.readVInt(); + List snapshots = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + snapshots.add(new SnapshotInfo(in)); + } + this.successfulResponses = Collections.singletonMap("unknown", snapshots); + this.failedResponses = Collections.emptyMap(); + } + } + public static class Response { private String repository; @@ -93,26 +127,23 @@ private static Response fromXContent(XContentParser parser) throws IOException { } } - private Map> successfulResponses = Collections.emptyMap(); - private Map failedResponses = Collections.emptyMap(); + private final Map> successfulResponses; + private final Map failedResponses; public GetSnapshotsResponse(Collection responses) { - this.successfulResponses = new HashMap<>(); - this.failedResponses = new HashMap<>(); + Map> successfulResponses = new HashMap<>(); + Map failedResponses = new HashMap<>(); for (Response response : responses) { if (response.snapshots != null) { assert response.error == null; - this.successfulResponses.put(response.repository, response.snapshots); + successfulResponses.put(response.repository, response.snapshots); } else { assert response.snapshots == null; - this.failedResponses.put(response.repository, response.error); + failedResponses.put(response.repository, response.error); } } - successfulResponses = Collections.unmodifiableMap(successfulResponses); - failedResponses = Collections.unmodifiableMap(failedResponses); - } - - public GetSnapshotsResponse() { + this.successfulResponses = Collections.unmodifiableMap(successfulResponses); + this.failedResponses = Collections.unmodifiableMap(failedResponses); } /** @@ -229,37 +260,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - if (in.getVersion().onOrAfter(GetSnapshotsRequest.MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) { - int successfulSize = in.readVInt(); - successfulResponses = new HashMap<>(successfulSize); - for (int i = 0; i < successfulSize; i++) { - String repository = in.readString(); - int size = in.readVInt(); - List snapshotInfos = new ArrayList<>(size); - for (int j = 0; j < size; j++) { - snapshotInfos.add(new SnapshotInfo(in)); - } - successfulResponses.put(repository, snapshotInfos); - } - - int failedSize = in.readVInt(); - failedResponses = new HashMap<>(failedSize); - for (int i = 0; i < failedSize; i++) { - String repository = in.readString(); - ElasticsearchException error = in.readException(); - failedResponses.put(repository, error); - } - successfulResponses = Collections.unmodifiableMap(successfulResponses); - failedResponses = Collections.unmodifiableMap(failedResponses); - } else { - int size = in.readVInt(); - List snapshots = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - snapshots.add(new SnapshotInfo(in)); - } - successfulResponses = Collections.singletonMap("unknown", snapshots); - } + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); } public static GetSnapshotsResponse fromXContent(XContentParser parser) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 7ae8aca4b278a..246b9c8fe132c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -37,6 +37,7 @@ import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoryData; @@ -47,6 +48,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -78,7 +80,12 @@ protected String executor() { @Override protected GetSnapshotsResponse newResponse() { - return new GetSnapshotsResponse(); + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + protected GetSnapshotsResponse read(StreamInput in) throws IOException { + return new GetSnapshotsResponse(in); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java index 31b55456c9c0f..081db2e5383c9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java @@ -59,7 +59,9 @@ private GetSnapshotsResponse doParseInstance(XContentParser parser) throws IOExc } private GetSnapshotsResponse copyInstance(GetSnapshotsResponse instance, Version version) throws IOException { - return copyStreamable(instance, new NamedWriteableRegistry(Collections.emptyList()), () -> new GetSnapshotsResponse(), version); + return copyInstance(instance, new NamedWriteableRegistry(Collections.emptyList()), (out, value) -> value.writeTo(out), + in -> new GetSnapshotsResponse(in), version); + } private void assertEqualInstances(GetSnapshotsResponse expectedInstance, GetSnapshotsResponse newInstance) { From c6435a12188519204755e91a2f4f4a0a6fdb26f8 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 17 Jun 2019 18:26:43 +0300 Subject: [PATCH 35/35] Fix HDFS yml tests --- .../test/hdfs_repository/30_snapshot_get.yml | 8 ++++---- .../test/hdfs_repository/30_snapshot_readonly.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml index f38f4783b195b..6ed93e07160b2 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml @@ -46,8 +46,8 @@ repository: test_snapshot_get_repository snapshot: test_snapshot_get - - length: { snapshots: 1 } - - match: { snapshots.0.snapshot : test_snapshot_get } + - length: { responses.0.snapshots: 1 } + - match: { responses.0.snapshots.0.snapshot : test_snapshot_get } # List snapshot info - do: @@ -55,8 +55,8 @@ repository: test_snapshot_get_repository snapshot: "*" - - length: { snapshots: 1 } - - match: { snapshots.0.snapshot : test_snapshot_get } + - length: { responses.0.snapshots: 1 } + - match: { responses.0.snapshots.0.snapshot : test_snapshot_get } # Remove our snapshot - do: diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml index c2a37964e70a7..dda910ae36c26 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml @@ -21,7 +21,7 @@ repository: test_snapshot_repository_ro snapshot: "_all" - - length: { snapshots: 1 } + - length: { responses.0.snapshots: 1 } # Remove our repository - do: