Add S3 integration tests for searchable snapshots (#52263)

This commit adds REST-based integration tests that test the 
searchable snapshots feature on S3.

Author: tlrx

plugins/repository-s3/build.gradle

@@ -214,7 +214,12 @@ processTestResources {
   MavenFilteringHack.filter(it, expansions)
 }
 
-testFixtures.useFixture(':test:fixtures:s3-fixture')
+[
+  's3-fixture',
+  's3-fixture-with-session-token',
+  's3-fixture-with-ec2',
+  's3-fixture-with-ecs',
+].forEach { fixture -> testFixtures.useFixture(':test:fixtures:s3-fixture', fixture) }
 
 def fixtureAddress = { fixture ->
   assert useFixture: 'closure should not be used without a fixture'

test/fixtures/s3-fixture/docker-compose.yml

@@ -15,6 +15,21 @@ services:
     ports:
       - "80"
 
+  s3-fixture-other:
+    build:
+      context: .
+      args:
+        fixtureClass: fixture.s3.S3HttpFixture
+        port: 80
+        bucket: "bucket"
+        basePath: "base_path"
+        accessKey: "access_key"
+      dockerfile: Dockerfile
+    volumes:
+      - ./testfixtures_shared/shared:/fixture/shared
+    ports:
+      - "80"
+
   s3-fixture-with-session-token:
     build:
       context: .

test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java

@@ -216,13 +216,13 @@ public void handle(final HttpExchange exchange) throws IOException {
 
                         final int start = Integer.parseInt(matcher.group(1));
                         final int end = Integer.parseInt(matcher.group(2));
-                        final int length = end - start;
 
+                        final BytesReference rangeBlob = blob.slice(start, end + 1 - start);
                         exchange.getResponseHeaders().add("Content-Type", "application/octet-stream");
-                        exchange.getResponseHeaders().add("Content-Range",
-                            String.format(Locale.ROOT, "bytes=%d-%d/%d", start, end, blob.length()));
-                        exchange.sendResponseHeaders(RestStatus.OK.getStatus(), length);
-                        exchange.getResponseBody().write(BytesReference.toBytes(blob), start, length);
+                        exchange.getResponseHeaders().add("Content-Range", String.format(Locale.ROOT, "bytes %d-%d/%d",
+                            start, end, rangeBlob.length()));
+                        exchange.sendResponseHeaders(RestStatus.OK.getStatus(), rangeBlob.length());
+                        rangeBlob.writeTo(exchange.getResponseBody());
                     }
                 } else {
                     exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1);

x-pack/plugin/searchable-snapshots/build.gradle

@@ -27,3 +27,16 @@ gradle.projectsEvaluated {
     .findAll { it.path.startsWith(project.path + ":qa") }
     .each { check.dependsOn it.check }
 }
+
+configurations {
+  testArtifacts.extendsFrom testRuntime
+}
+
+task testJar(type: Jar) {
+  appendix 'test'
+  from sourceSets.test.output
+}
+
+artifacts {
+  testArtifacts testJar
+}

x-pack/plugin/searchable-snapshots/qa/rest/build.gradle

@@ -2,7 +2,17 @@ apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
+dependencies {
+  testCompile project(path: xpackModule('searchable-snapshots'), configuration: 'testArtifacts')
+}
+
+final File repoDir = file("$buildDir/testclusters/repo")
+
+integTest.runner {
+  systemProperty 'tests.path.repo', repoDir
+}
+
 testClusters.integTest {
   testDistribution = 'DEFAULT'
-  setting 'xpack.license.self_generated.type', 'basic'
+  setting 'path.repo', repoDir.absolutePath
 }

x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/FsSearchableSnapshotsIT.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.searchablesnapshots.rest;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsRestTestCase;
+
+public class FsSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase {
+
+    @Override
+    protected String repositoryType() {
+        return FsRepository.TYPE;
+    }
+
+    @Override
+    protected Settings repositorySettings() {
+        final Settings.Builder settings = Settings.builder();
+        settings.put("location", System.getProperty("tests.path.repo"));
+        if (randomBoolean()) {
+            settings.put("compress", randomBoolean());
+        }
+        if (randomBoolean()) {
+            settings.put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES);
+        }
+        return settings.build();
+    }
+}

x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/SearchableSnapshotsRestIT.java renamed to x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/SearchableSnapshotsClientYamlTestSuiteIT.java

@@ -3,16 +3,15 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-
 package org.elasticsearch.xpack.searchablesnapshots.rest;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-public class SearchableSnapshotsRestIT extends ESClientYamlSuiteTestCase {
+public class SearchableSnapshotsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
-    public SearchableSnapshotsRestIT(final ClientYamlTestCandidate testCandidate) {
+    public SearchableSnapshotsClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
 

x-pack/plugin/searchable-snapshots/qa/s3/build.gradle

@@ -0,0 +1,66 @@
+import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+final Project fixture = project(':test:fixtures:s3-fixture')
+final Project repositoryPlugin = project(':plugins:repository-s3')
+
+dependencies {
+  testCompile project(path: xpackModule('searchable-snapshots'), configuration: 'testArtifacts')
+  testCompile repositoryPlugin
+}
+
+boolean useFixture = false
+String s3AccessKey = System.getenv("amazon_s3_access_key")
+String s3SecretKey = System.getenv("amazon_s3_secret_key")
+String s3Bucket = System.getenv("amazon_s3_bucket")
+String s3BasePath = System.getenv("amazon_s3_base_path")
+
+if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) {
+  s3AccessKey = 'access_key'
+  s3SecretKey = 'secret_key'
+  s3Bucket = 'bucket'
+  s3BasePath = 'base_path'
+  useFixture = true
+
+} else if (!s3AccessKey || !s3SecretKey || !s3Bucket || !s3BasePath) {
+  throw new IllegalArgumentException("not all options specified to run against external S3 service are present")
+}
+
+if (useFixture) {
+  apply plugin: 'elasticsearch.test.fixtures'
+  testFixtures.useFixture(fixture.path, 's3-fixture-other')
+}
+
+integTest {
+  dependsOn repositoryPlugin.bundlePlugin
+  runner {
+    systemProperty 'test.s3.bucket', s3Bucket
+    systemProperty 'test.s3.base_path', s3BasePath + "/searchable_snapshots_tests"
+  }
+}
+
+testClusters.integTest {
+  testDistribution = 'DEFAULT'
+  plugin file(repositoryPlugin.bundlePlugin.archiveFile)
+
+  keystore 's3.client.searchable_snapshots.access_key', s3AccessKey
+  keystore 's3.client.searchable_snapshots.secret_key', s3SecretKey
+
+  if (useFixture) {
+    def fixtureAddress = { fixtureName ->
+      assert useFixture: 'closure should not be used without a fixture'
+      int ephemeralPort =  fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.80"
+      assert ephemeralPort > 0
+      '127.0.0.1:' + ephemeralPort
+    }
+
+    setting 's3.client.searchable_snapshots.protocol', 'http'
+    setting 's3.client.searchable_snapshots.endpoint', { "${-> fixtureAddress('s3-fixture-other')}" }, IGNORE_VALUE
+
+  } else {
+    println "Using an external service to test " + project.name
+  }
+}
+

x-pack/plugin/searchable-snapshots/qa/s3/src/test/java/org/elasticsearch/xpack/searchablesnapshots/s3/S3SearchableSnapshotsIT.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.searchablesnapshots.s3;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsRestTestCase;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.not;
+
+public class S3SearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase {
+
+    @Override
+    protected String repositoryType() {
+        return "s3";
+    }
+
+    @Override
+    protected Settings repositorySettings() {
+        final String bucket = System.getProperty("test.s3.bucket");
+        assertThat(bucket, not(blankOrNullString()));
+
+        final String basePath = System.getProperty("test.s3.base_path");
+        assertThat(basePath, not(blankOrNullString()));
+
+        return Settings.builder()
+            .put("client", "searchable_snapshots")
+            .put("bucket", bucket)
+            .put("base_path", basePath)
+            .build();
+    }
+}