From 6a646000c06536d0087d0bdfaf4a1c612e8ea17c Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 22 Jul 2020 11:44:13 -0500 Subject: [PATCH 1/5] repository-azure --- plugins/repository-azure/build.gradle | 16 +++++++++++----- .../azure/AzureBlobStoreRepositoryTests.java | 0 .../AzureStorageCleanupThirdPartyTests.java | 0 .../RepositoryAzureClientYamlTestSuiteIT.java | 0 .../test/repository_azure/10_basic.yml | 0 .../test/repository_azure/20_repository.yml | 0 6 files changed, 11 insertions(+), 5 deletions(-) rename plugins/repository-azure/src/{test => internalClusterTest}/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java (100%) rename plugins/repository-azure/src/{test => internalClusterTest}/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java (100%) rename plugins/repository-azure/src/{test => yamlRestTest}/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java (100%) rename plugins/repository-azure/src/{test => yamlRestTest}/resources/rest-api-spec/test/repository_azure/10_basic.yml (100%) rename plugins/repository-azure/src/{test => yamlRestTest}/resources/rest-api-spec/test/repository_azure/20_repository.yml (100%) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 723a4c1e0753a..6cfae235d0ab5 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -1,5 +1,6 @@ import org.elasticsearch.gradle.MavenFilteringHack import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.test.InternalClusterTestPlugin import static org.elasticsearch.gradle.PropertyNormalization.DEFAULT import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE @@ -22,7 +23,8 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'The Azure Repository plugin adds support for Azure storage repositories.' @@ -106,18 +108,18 @@ Map expansions = [ 'base_path': azureBasePath + "_integration_tests" ] -processTestResources { +processYamlRestTestResources { inputs.properties(expansions) MavenFilteringHack.filter(it, expansions) } -test { +internalClusterTest { // this is tested explicitly in a separate test task exclude '**/AzureStorageCleanupThirdPartyTests.class' } testClusters { - integTest { + yamlRestTest { keystore 'azure.client.integration_test.account', azureAccount if (azureKey != null && azureKey.isEmpty() == false) { keystore 'azure.client.integration_test.key', azureKey @@ -134,7 +136,11 @@ testClusters { } task azureThirdPartyTest(type: Test) { - dependsOn tasks.integTest + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs()) + dependsOn tasks.internalClusterTest + setClasspath(internalTestSourceSet.getRuntimeClasspath()) include '**/AzureStorageCleanupThirdPartyTests.class' systemProperty 'test.azure.account', azureAccount ? azureAccount : "" systemProperty 'test.azure.key', azureKey ? azureKey : "" diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java similarity index 100% rename from plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java rename to plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java similarity index 100% rename from plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java rename to plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java b/plugins/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java similarity index 100% rename from plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java rename to plugins/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java diff --git a/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/10_basic.yml b/plugins/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/10_basic.yml similarity index 100% rename from plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/10_basic.yml rename to plugins/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/10_basic.yml diff --git a/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml b/plugins/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml similarity index 100% rename from plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml rename to plugins/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml From fa0bd0d80851bcf04f155e77979fb20b169ca011 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 22 Jul 2020 12:09:39 -0500 Subject: [PATCH 2/5] repository-gcs --- plugins/repository-azure/build.gradle | 2 +- plugins/repository-gcs/build.gradle | 36 ++++++++++++------- ...eCloudStorageBlobStoreRepositoryTests.java | 0 .../GoogleCloudStorageThirdPartyTests.java | 0 .../RepositoryGcsClientYamlTestSuiteIT.java | 0 .../test/repository_gcs/10_basic.yml | 0 .../test/repository_gcs/20_repository.yml | 0 7 files changed, 25 insertions(+), 13 deletions(-) rename plugins/repository-gcs/src/{test => internalClusterTest}/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java (100%) rename plugins/repository-gcs/src/{test => internalClusterTest}/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java (100%) rename plugins/repository-gcs/src/{test => yamlRestTest}/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java (100%) rename plugins/repository-gcs/src/{test => yamlRestTest}/resources/rest-api-spec/test/repository_gcs/10_basic.yml (100%) rename plugins/repository-gcs/src/{test => yamlRestTest}/resources/rest-api-spec/test/repository_gcs/20_repository.yml (100%) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 6cfae235d0ab5..308e4ab59bf42 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -139,8 +139,8 @@ task azureThirdPartyTest(type: Test) { SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME) setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs()) - dependsOn tasks.internalClusterTest setClasspath(internalTestSourceSet.getRuntimeClasspath()) + dependsOn tasks.internalClusterTest include '**/AzureStorageCleanupThirdPartyTests.class' systemProperty 'test.azure.account', azureAccount ? azureAccount : "" systemProperty 'test.azure.key', azureKey ? azureKey : "" diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 5a60f96d8abbd..7b6ff4be342b0 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -4,6 +4,8 @@ import java.security.KeyPairGenerator import org.elasticsearch.gradle.MavenFilteringHack import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.test.RestIntegTestTask +import org.elasticsearch.gradle.test.rest.YamlRestTestPlugin +import org.elasticsearch.gradle.test.InternalClusterTestPlugin import java.nio.file.Files import java.security.KeyPair @@ -28,7 +30,8 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'The GCS repository plugin adds Google Cloud Storage support for repositories.' @@ -254,12 +257,12 @@ Map expansions = [ 'base_path': gcsBasePath + "_integration_tests" ] -processTestResources { +processYamlRestTestResources { inputs.properties(expansions) MavenFilteringHack.filter(it, expansions) } -test { +internalClusterTest { // this is tested explicitly in a separate test task exclude '**/GoogleCloudStorageThirdPartyTests.class' } @@ -276,33 +279,39 @@ final Closure testClustersConfiguration = { } } -integTest { +yamlRestTest { if (useFixture) { dependsOn createServiceAccountFile } } -check.dependsOn integTest testClusters { - integTest testClustersConfiguration + all testClustersConfiguration } /* * We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add * an additional test that forces the large blob threshold to be small to exercise the resumable upload path. */ -task largeBlobIntegTest(type: RestIntegTestTask) { - mustRunAfter integTest +task largeBlobYamlRestTest(type: RestIntegTestTask) { + mustRunAfter yamlRestTest dependsOn project(':plugins:repository-gcs').bundlePlugin if (useFixture) { dependsOn createServiceAccountFile } } -check.dependsOn largeBlobIntegTest +largeBlobYamlRestTest { + runner { + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) + setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) + } +} +check.dependsOn largeBlobYamlRestTest -testClusters.largeBlobIntegTest testClustersConfiguration testClusters { - largeBlobIntegTest { + largeBlobYamlRestTest { plugin project(':plugins:repository-gcs').bundlePlugin.archiveFile // force large blob uploads by setting the threshold small, forcing this code path to be tested @@ -311,7 +320,10 @@ testClusters { } task gcsThirdPartyTest(type: Test) { - dependsOn integTest,largeBlobIntegTest + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs()) + setClasspath(internalTestSourceSet.getRuntimeClasspath()) include '**/GoogleCloudStorageThirdPartyTests.class' systemProperty 'tests.security.manager', false systemProperty 'test.google.bucket', gcsBucket diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java similarity index 100% rename from plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java rename to plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java b/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java similarity index 100% rename from plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java rename to plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java b/plugins/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java similarity index 100% rename from plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java rename to plugins/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java diff --git a/plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/10_basic.yml b/plugins/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/10_basic.yml similarity index 100% rename from plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/10_basic.yml rename to plugins/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/10_basic.yml diff --git a/plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/20_repository.yml b/plugins/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml similarity index 100% rename from plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/20_repository.yml rename to plugins/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml From 9aba0329cf145e36f3dff439d5c4839da2901389 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 22 Jul 2020 13:37:09 -0500 Subject: [PATCH 3/5] repository-s3 --- plugins/repository-gcs/build.gradle | 5 +-- plugins/repository-s3/build.gradle | 45 +++++++++++++------ .../s3/S3BlobStoreRepositoryTests.java | 3 +- .../s3/S3RepositoryThirdPartyTests.java | 0 .../s3/RepositoryS3ClientYamlTestSuiteIT.java | 0 .../test/repository_s3/10_basic.yml | 0 .../20_repository_permanent_credentials.yml | 0 .../30_repository_temporary_credentials.yml | 0 .../40_repository_ec2_credentials.yml | 0 .../50_repository_ecs_credentials.yml | 0 10 files changed, 35 insertions(+), 18 deletions(-) rename plugins/repository-s3/src/{test => internalClusterTest}/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java (99%) rename plugins/repository-s3/src/{test => internalClusterTest}/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java (100%) rename plugins/repository-s3/src/{test => yamlRestTest}/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java (100%) rename plugins/repository-s3/src/{test => yamlRestTest}/resources/rest-api-spec/test/repository_s3/10_basic.yml (100%) rename plugins/repository-s3/src/{test => yamlRestTest}/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml (100%) rename plugins/repository-s3/src/{test => yamlRestTest}/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml (100%) rename plugins/repository-s3/src/{test => yamlRestTest}/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml (100%) rename plugins/repository-s3/src/{test => yamlRestTest}/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml (100%) diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 7b6ff4be342b0..24a9185c5c218 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -294,13 +294,11 @@ testClusters { * an additional test that forces the large blob threshold to be small to exercise the resumable upload path. */ task largeBlobYamlRestTest(type: RestIntegTestTask) { - mustRunAfter yamlRestTest + mustRunAfter yamlRestTest //to allow maven style filtering dependsOn project(':plugins:repository-gcs').bundlePlugin if (useFixture) { dependsOn createServiceAccountFile } -} -largeBlobYamlRestTest { runner { SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) @@ -308,6 +306,7 @@ largeBlobYamlRestTest { setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) } } + check.dependsOn largeBlobYamlRestTest testClusters { diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 38362209228e6..007b4df77aa5a 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -1,6 +1,8 @@ import org.elasticsearch.gradle.MavenFilteringHack import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.test.RestIntegTestTask +import org.elasticsearch.gradle.test.rest.YamlRestTestPlugin +import org.elasticsearch.gradle.test.InternalClusterTestPlugin import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE @@ -22,7 +24,8 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'The S3 repository plugin adds S3 repositories' @@ -135,7 +138,7 @@ if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) { throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present") } -processTestResources { +processYamlRestTestResources { Map expansions = [ 'permanent_bucket' : s3PermanentBucket, 'permanent_base_path' : s3PermanentBasePath + "_integration_tests", @@ -151,13 +154,12 @@ processTestResources { MavenFilteringHack.filter(it, expansions) } -test { +internalClusterTest { // this is tested explicitly in a separate test task exclude '**/S3RepositoryThirdPartyTests.class' } -// IntegTest -integTest { +yamlRestTest { runner { systemProperty 'tests.rest.blacklist', ( useFixture ? @@ -172,7 +174,7 @@ integTest { } } -testClusters.integTest { +testClusters.yamlRestTest { keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey @@ -207,10 +209,16 @@ testClusters.integTest { if (useFixture) { testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture') - task integTestMinio(type: RestIntegTestTask) { + task yamlRestTestMinio(type: RestIntegTestTask) { + mustRunAfter 'yamlRestTest' //to allow maven style filtering description = "Runs REST tests using the Minio repository." dependsOn tasks.bundlePlugin runner { + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) + setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) + // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968 systemProperty 'tests.rest.blacklist', [ 'repository_s3/30_repository_temporary_credentials/*', @@ -219,9 +227,9 @@ if (useFixture) { ].join(",") } } - check.dependsOn(integTestMinio) + check.dependsOn(yamlRestTestMinio) - testClusters.integTestMinio { + testClusters.yamlRestTestMinio { keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000')}" }, IGNORE_VALUE @@ -233,10 +241,15 @@ if (useFixture) { if (useFixture) { testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ecs') - task integTestECS(type: RestIntegTestTask.class) { + task yamlRestTestECS(type: RestIntegTestTask.class) { + mustRunAfter 'yamlRestTest' //to allow maven style filtering description = "Runs tests using the ECS repository." dependsOn('bundlePlugin') runner { + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) + setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) systemProperty 'tests.rest.blacklist', [ 'repository_s3/10_basic/*', 'repository_s3/20_repository_permanent_credentials/*', @@ -245,9 +258,9 @@ if (useFixture) { ].join(",") } } - check.dependsOn(integTestECS) + check.dependsOn(yamlRestTestECS) - testClusters.integTestECS { + testClusters.yamlRestTestECS { setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}" }, IGNORE_VALUE plugin tasks.bundlePlugin.archiveFile environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}/ecs_credentials_endpoint" }, IGNORE_VALUE @@ -256,16 +269,20 @@ if (useFixture) { // 3rd Party Tests task s3ThirdPartyTest(type: Test) { + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs()) + setClasspath(internalTestSourceSet.getRuntimeClasspath()) include '**/S3RepositoryThirdPartyTests.class' systemProperty 'test.s3.account', s3PermanentAccessKey systemProperty 'test.s3.key', s3PermanentSecretKey systemProperty 'test.s3.bucket', s3PermanentBucket nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed if (useFixture) { - dependsOn tasks.integTestMinio + // dependsOn tasks.yamlRestTestMinio nonInputProperties.systemProperty 'test.s3.endpoint', "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000') }" } else { - dependsOn tasks.integTest + // dependsOn tasks.yamlRestTest } } check.dependsOn(s3ThirdPartyTest) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java similarity index 99% rename from plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java rename to plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index d54181001c0fa..3bf452cdb949b 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -304,3 +304,4 @@ private boolean isMultiPartUpload(String request) { } } } + diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java similarity index 100% rename from plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java rename to plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java b/plugins/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java similarity index 100% rename from plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java rename to plugins/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/10_basic.yml b/plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/10_basic.yml similarity index 100% rename from plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/10_basic.yml rename to plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/10_basic.yml diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml similarity index 100% rename from plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml rename to plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml b/plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml similarity index 100% rename from plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml rename to plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml similarity index 100% rename from plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml rename to plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml similarity index 100% rename from plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml rename to plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml From 7a200253be03eaca464ebb7bd3f3671652387933 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 27 Jul 2020 14:46:44 -0500 Subject: [PATCH 4/5] minor fixes for s3 --- plugins/repository-s3/build.gradle | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 007b4df77aa5a..b7f96fe5d0a2e 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -58,6 +58,12 @@ dependencies { testImplementation project(':test:fixtures:s3-fixture') } +restResources { + restApi { + includeCore '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count' + } +} + tasks.named("dependencyLicenses").configure { mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk' mapping from: /jmespath-java.*/, to: 'aws-java-sdk' @@ -210,7 +216,6 @@ if (useFixture) { testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture') task yamlRestTestMinio(type: RestIntegTestTask) { - mustRunAfter 'yamlRestTest' //to allow maven style filtering description = "Runs REST tests using the Minio repository." dependsOn tasks.bundlePlugin runner { @@ -240,9 +245,7 @@ if (useFixture) { // ECS if (useFixture) { testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ecs') - task yamlRestTestECS(type: RestIntegTestTask.class) { - mustRunAfter 'yamlRestTest' //to allow maven style filtering description = "Runs tests using the ECS repository." dependsOn('bundlePlugin') runner { @@ -279,10 +282,7 @@ task s3ThirdPartyTest(type: Test) { systemProperty 'test.s3.bucket', s3PermanentBucket nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed if (useFixture) { - // dependsOn tasks.yamlRestTestMinio nonInputProperties.systemProperty 'test.s3.endpoint', "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000') }" - } else { - // dependsOn tasks.yamlRestTest } } check.dependsOn(s3ThirdPartyTest) From bbaaeec9c37d2fdd007334f96ae3dd7671e7ad9d Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 27 Jul 2020 16:45:33 -0500 Subject: [PATCH 5/5] minor gcs --- plugins/repository-gcs/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 24a9185c5c218..e4803dccdd527 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -294,7 +294,6 @@ testClusters { * an additional test that forces the large blob threshold to be small to exercise the resumable upload path. */ task largeBlobYamlRestTest(type: RestIntegTestTask) { - mustRunAfter yamlRestTest //to allow maven style filtering dependsOn project(':plugins:repository-gcs').bundlePlugin if (useFixture) { dependsOn createServiceAccountFile