diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 34660f6b2b54b..28d164989f86b 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -214,7 +214,12 @@ processTestResources { MavenFilteringHack.filter(it, expansions) } -testFixtures.useFixture(':test:fixtures:s3-fixture') +[ + 's3-fixture', + 's3-fixture-with-session-token', + 's3-fixture-with-ec2', + 's3-fixture-with-ecs', +].forEach { fixture -> testFixtures.useFixture(':test:fixtures:s3-fixture', fixture) } def fixtureAddress = { fixture -> assert useFixture: 'closure should not be used without a fixture' diff --git a/test/fixtures/s3-fixture/docker-compose.yml b/test/fixtures/s3-fixture/docker-compose.yml index 401a43c9255b7..1d06334eddbd3 100644 --- a/test/fixtures/s3-fixture/docker-compose.yml +++ b/test/fixtures/s3-fixture/docker-compose.yml @@ -15,6 +15,21 @@ services: ports: - "80" + s3-fixture-other: + build: + context: . + args: + fixtureClass: fixture.s3.S3HttpFixture + port: 80 + bucket: "bucket" + basePath: "base_path" + accessKey: "access_key" + dockerfile: Dockerfile + volumes: + - ./testfixtures_shared/shared:/fixture/shared + ports: + - "80" + s3-fixture-with-session-token: build: context: . diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java index f9bce9f02c85d..7ae8747aadb2f 100644 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java @@ -216,13 +216,13 @@ public void handle(final HttpExchange exchange) throws IOException { final int start = Integer.parseInt(matcher.group(1)); final int end = Integer.parseInt(matcher.group(2)); - final int length = end - start; + final BytesReference rangeBlob = blob.slice(start, end + 1 - start); exchange.getResponseHeaders().add("Content-Type", "application/octet-stream"); - exchange.getResponseHeaders().add("Content-Range", - String.format(Locale.ROOT, "bytes=%d-%d/%d", start, end, blob.length())); - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), length); - exchange.getResponseBody().write(BytesReference.toBytes(blob), start, length); + exchange.getResponseHeaders().add("Content-Range", String.format(Locale.ROOT, "bytes %d-%d/%d", + start, end, rangeBlob.length())); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), rangeBlob.length()); + rangeBlob.writeTo(exchange.getResponseBody()); } } else { exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1); diff --git a/x-pack/plugin/searchable-snapshots/build.gradle b/x-pack/plugin/searchable-snapshots/build.gradle index dcb22ec38a51f..0ed4e994af266 100644 --- a/x-pack/plugin/searchable-snapshots/build.gradle +++ b/x-pack/plugin/searchable-snapshots/build.gradle @@ -27,3 +27,16 @@ gradle.projectsEvaluated { .findAll { it.path.startsWith(project.path + ":qa") } .each { check.dependsOn it.check } } + +configurations { + testArtifacts.extendsFrom testRuntime +} + +task testJar(type: Jar) { + appendix 'test' + from sourceSets.test.output +} + +artifacts { + testArtifacts testJar +} diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle index 5a14efb38f466..3b1308e51b0bd 100644 --- a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle @@ -2,7 +2,17 @@ apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' +dependencies { + testCompile project(path: xpackModule('searchable-snapshots'), configuration: 'testArtifacts') +} + +final File repoDir = file("$buildDir/testclusters/repo") + +integTest.runner { + systemProperty 'tests.path.repo', repoDir +} + testClusters.integTest { testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'basic' + setting 'path.repo', repoDir.absolutePath } diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/FsSearchableSnapshotsIT.java b/x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/FsSearchableSnapshotsIT.java new file mode 100644 index 0000000000000..a3efec0c8f2f3 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/FsSearchableSnapshotsIT.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.searchablesnapshots.rest; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsRestTestCase; + +public class FsSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase { + + @Override + protected String repositoryType() { + return FsRepository.TYPE; + } + + @Override + protected Settings repositorySettings() { + final Settings.Builder settings = Settings.builder(); + settings.put("location", System.getProperty("tests.path.repo")); + if (randomBoolean()) { + settings.put("compress", randomBoolean()); + } + if (randomBoolean()) { + settings.put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES); + } + return settings.build(); + } +} diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/SearchableSnapshotsRestIT.java b/x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/SearchableSnapshotsClientYamlTestSuiteIT.java similarity index 78% rename from x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/SearchableSnapshotsRestIT.java rename to x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/SearchableSnapshotsClientYamlTestSuiteIT.java index 025f594ef669a..04e420bfba551 100644 --- a/x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/SearchableSnapshotsRestIT.java +++ b/x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/SearchableSnapshotsClientYamlTestSuiteIT.java @@ -3,16 +3,15 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - package org.elasticsearch.xpack.searchablesnapshots.rest; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -public class SearchableSnapshotsRestIT extends ESClientYamlSuiteTestCase { +public class SearchableSnapshotsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { - public SearchableSnapshotsRestIT(final ClientYamlTestCandidate testCandidate) { + public SearchableSnapshotsClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { super(testCandidate); } diff --git a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle new file mode 100644 index 0000000000000..58eb3e288a485 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle @@ -0,0 +1,66 @@ +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE + +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' + +final Project fixture = project(':test:fixtures:s3-fixture') +final Project repositoryPlugin = project(':plugins:repository-s3') + +dependencies { + testCompile project(path: xpackModule('searchable-snapshots'), configuration: 'testArtifacts') + testCompile repositoryPlugin +} + +boolean useFixture = false +String s3AccessKey = System.getenv("amazon_s3_access_key") +String s3SecretKey = System.getenv("amazon_s3_secret_key") +String s3Bucket = System.getenv("amazon_s3_bucket") +String s3BasePath = System.getenv("amazon_s3_base_path") + +if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) { + s3AccessKey = 'access_key' + s3SecretKey = 'secret_key' + s3Bucket = 'bucket' + s3BasePath = 'base_path' + useFixture = true + +} else if (!s3AccessKey || !s3SecretKey || !s3Bucket || !s3BasePath) { + throw new IllegalArgumentException("not all options specified to run against external S3 service are present") +} + +if (useFixture) { + apply plugin: 'elasticsearch.test.fixtures' + testFixtures.useFixture(fixture.path, 's3-fixture-other') +} + +integTest { + dependsOn repositoryPlugin.bundlePlugin + runner { + systemProperty 'test.s3.bucket', s3Bucket + systemProperty 'test.s3.base_path', s3BasePath + "/searchable_snapshots_tests" + } +} + +testClusters.integTest { + testDistribution = 'DEFAULT' + plugin file(repositoryPlugin.bundlePlugin.archiveFile) + + keystore 's3.client.searchable_snapshots.access_key', s3AccessKey + keystore 's3.client.searchable_snapshots.secret_key', s3SecretKey + + if (useFixture) { + def fixtureAddress = { fixtureName -> + assert useFixture: 'closure should not be used without a fixture' + int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.80" + assert ephemeralPort > 0 + '127.0.0.1:' + ephemeralPort + } + + setting 's3.client.searchable_snapshots.protocol', 'http' + setting 's3.client.searchable_snapshots.endpoint', { "${-> fixtureAddress('s3-fixture-other')}" }, IGNORE_VALUE + + } else { + println "Using an external service to test " + project.name + } +} + diff --git a/x-pack/plugin/searchable-snapshots/qa/s3/src/test/java/org/elasticsearch/xpack/searchablesnapshots/s3/S3SearchableSnapshotsIT.java b/x-pack/plugin/searchable-snapshots/qa/s3/src/test/java/org/elasticsearch/xpack/searchablesnapshots/s3/S3SearchableSnapshotsIT.java new file mode 100644 index 0000000000000..9a5eb98ca4c9a --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/qa/s3/src/test/java/org/elasticsearch/xpack/searchablesnapshots/s3/S3SearchableSnapshotsIT.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.searchablesnapshots.s3; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsRestTestCase; + +import static org.hamcrest.Matchers.blankOrNullString; +import static org.hamcrest.Matchers.not; + +public class S3SearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase { + + @Override + protected String repositoryType() { + return "s3"; + } + + @Override + protected Settings repositorySettings() { + final String bucket = System.getProperty("test.s3.bucket"); + assertThat(bucket, not(blankOrNullString())); + + final String basePath = System.getProperty("test.s3.base_path"); + assertThat(basePath, not(blankOrNullString())); + + return Settings.builder() + .put("client", "searchable_snapshots") + .put("bucket", bucket) + .put("base_path", basePath) + .build(); + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java new file mode 100644 index 0000000000000..25e1470cad69f --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java @@ -0,0 +1,225 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.searchablesnapshots; + +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +public abstract class AbstractSearchableSnapshotsRestTestCase extends ESRestTestCase { + + protected abstract String repositoryType(); + + protected abstract Settings repositorySettings(); + + public void testSearchableSnapshots() throws Exception { + final String repositoryType = repositoryType(); + final Settings repositorySettings = repositorySettings(); + + final String repository = "repository"; + logger.info("creating repository [{}] of type [{}]", repository, repositoryType); + registerRepository(repository, repositoryType, true, repositorySettings); + + final String searchableSnapshotRepository = "repository-searchable-snapshots"; + logger.info("creating searchable snapshots repository [{}]", searchableSnapshotRepository); + registerRepository(searchableSnapshotRepository, SearchableSnapshotRepository.TYPE, false, + Settings.builder().put("delegate_type", repositoryType).put("readonly", true).put(repositorySettings).build()); + + final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + logger.info("creating index [{}]", indexName); + createIndex(indexName, Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 5)) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .build()); + ensureGreen(indexName); + + final int numDocs = randomIntBetween(1, 10_000); + logger.info("indexing [{}] documents", numDocs); + + final StringBuilder bulkBody = new StringBuilder(); + for (int i = 0; i < numDocs; i++) { + bulkBody.append("{\"index\":{}}\n"); + bulkBody.append("{\"field\":").append(i).append(",\"text\":\"Document number ").append(i).append("\"}\n"); + } + + final Request documents = new Request(HttpPost.METHOD_NAME, '/' + indexName + "/_bulk"); + documents.addParameter("refresh", Boolean.TRUE.toString()); + documents.setJsonEntity(bulkBody.toString()); + assertOK(client().performRequest(documents)); + + logger.info("force merging index [{}]", indexName); + forceMerge(indexName, true, true); + + final String snapshot = "searchable-snapshot"; + + // Remove the snapshots, if a previous test failed to delete them. This is + // useful for third party tests that runs the test against a real external service. + deleteSnapshot(repository, snapshot, true); + + logger.info("creating snapshot [{}]", snapshot); + createSnapshot(repository, snapshot, true); + + logger.info("deleting index [{}]", indexName); + deleteIndex(indexName); + + final String restoredIndexName = randomBoolean() ? indexName : randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + logger.info("restoring index [{}] from snapshot [{}] as [{}]", indexName, snapshot, restoredIndexName); + restoreSnapshot(searchableSnapshotRepository, snapshot, true, indexName, restoredIndexName, Settings.EMPTY); + + ensureGreen(restoredIndexName); + + final Number count = count(restoredIndexName); + assertThat("Wrong index count for index " + restoredIndexName, count.intValue(), equalTo(numDocs)); + + for (int i = 0; i < 10; i++) { + final int randomTieBreaker = randomIntBetween(1, numDocs - 1); + Map searchResults; + switch (randomInt(3)) { + case 0: + searchResults = search(restoredIndexName, QueryBuilders.termQuery("field", String.valueOf(randomTieBreaker))); + assertThat(extractValue(searchResults, "hits.total.value"), equalTo(1)); + @SuppressWarnings("unchecked") + Map searchHit = (Map) ((List) extractValue(searchResults, "hits.hits")).get(0); + assertThat(extractValue(searchHit, "_index"), equalTo(restoredIndexName)); + assertThat(extractValue(searchHit, "_source.field"), equalTo(randomTieBreaker)); + break; + case 1: + searchResults = search(restoredIndexName, QueryBuilders.rangeQuery("field").lt(randomTieBreaker)); + assertThat(extractValue(searchResults, "hits.total.value"), equalTo(randomTieBreaker)); + break; + case 2: + searchResults = search(restoredIndexName, QueryBuilders.rangeQuery("field").gte(randomTieBreaker)); + assertThat(extractValue(searchResults, "hits.total.value"), equalTo(numDocs - randomTieBreaker)); + break; + case 3: + searchResults = search(restoredIndexName, QueryBuilders.matchQuery("text", "document")); + assertThat(extractValue(searchResults, "hits.total.value"), equalTo(numDocs)); + break; + default: + fail("Unsupported randomized search query"); + } + } + + logger.info("deleting snapshot [{}]", snapshot); + deleteSnapshot(repository, snapshot, false); + } + + protected static void registerRepository(String repository, String type, boolean verify, Settings settings) throws IOException { + final Request request = new Request(HttpPut.METHOD_NAME, "_snapshot/" + repository); + request.setJsonEntity(Strings.toString(new PutRepositoryRequest(repository).type(type).verify(verify).settings(settings))); + + final Response response = client().performRequest(request); + assertThat("Failed to create repository [" + repository + "] of type [" + type + "]: " + response, + response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); + } + + protected static void createSnapshot(String repository, String snapshot, boolean waitForCompletion) throws IOException { + final Request request = new Request(HttpPut.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot); + request.addParameter("wait_for_completion", Boolean.toString(waitForCompletion)); + + final Response response = client().performRequest(request); + assertThat("Failed to create snapshot [" + snapshot + "] in repository [" + repository + "]: " + response, + response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); + } + + protected static void deleteSnapshot(String repository, String snapshot, boolean ignoreMissing) throws IOException { + final Request request = new Request(HttpDelete.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot); + try { + final Response response = client().performRequest(request); + assertThat("Failed to delete snapshot [" + snapshot + "] in repository [" + repository + "]: " + response, + response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); + } catch (IOException e) { + if (ignoreMissing && e instanceof ResponseException) { + Response response = ((ResponseException) e).getResponse(); + assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.NOT_FOUND.getStatus())); + return; + } + throw e; + } + } + + protected static void restoreSnapshot(String repository, String snapshot, boolean waitForCompletion, + String renamePattern, String renameReplacement, Settings indexSettings) throws IOException { + final Request request = new Request(HttpPost.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot + "/_restore"); + request.addParameter("wait_for_completion", Boolean.toString(waitForCompletion)); + request.setJsonEntity(Strings.toString(new RestoreSnapshotRequest(repository, snapshot) + .renamePattern(renamePattern).renameReplacement(renameReplacement).indexSettings(indexSettings))); + + final Response response = client().performRequest(request); + assertThat("Failed to restore snapshot [" + snapshot + "] in repository [" + repository + "]: " + response, + response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); + } + + protected static void forceMerge(String index, boolean onlyExpungeDeletes, boolean flush) throws IOException { + final Request request = new Request(HttpPost.METHOD_NAME, '/' + index + "/_forcemerge"); + request.addParameter("only_expunge_deletes", Boolean.toString(onlyExpungeDeletes)); + request.addParameter("flush", Boolean.toString(flush)); + assertOK(client().performRequest(request)); + } + + protected static Number count(String index) throws IOException { + final Response response = client().performRequest(new Request(HttpPost.METHOD_NAME, '/' + index + "/_count")); + assertThat("Failed to execute count request on index [" + index + "]: " + response, + response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); + + final Map responseAsMap = responseAsMap(response); + assertThat("Shard failures when executing count request on index [" + index + "]: " + response, + extractValue(responseAsMap, "_shards.failed"), equalTo(0)); + return (Number) extractValue(responseAsMap, "count"); + } + + protected static Map search(String index, QueryBuilder query) throws IOException { + final Request request = new Request(HttpPost.METHOD_NAME, '/' + index + "/_search"); + request.setJsonEntity(new SearchSourceBuilder().trackTotalHits(true).query(query).toString()); + + final Response response = client().performRequest(request); + assertThat("Failed to execute search request on index [" + index + "]: " + response, + response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); + + final Map responseAsMap = responseAsMap(response); + assertThat("Shard failures when executing search request on index [" + index + "]: " + response, + extractValue(responseAsMap, "_shards.failed"), equalTo(0)); + return responseAsMap; + } + + protected static Map responseAsMap(Response response) throws IOException { + final XContentType xContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue()); + assertThat("Unknown XContentType", xContentType, notNullValue()); + try (InputStream responseBody = response.getEntity().getContent()) { + return XContentHelper.convertToMap(xContentType.xContent(), responseBody, true); + } + } + + @SuppressWarnings("unchecked") + protected static T extractValue(Map map, String path) { + return (T) XContentMapValues.extractValue(path, map); + } +}