From a65914c9d588d030a780b796f1c4aadce0768797 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 20 Jul 2020 17:30:22 -0500 Subject: [PATCH 1/5] new javaRestTest plugin and 1/2 modules converted --- .../gradle/plugin/PluginBuildPlugin.groovy | 6 + .../test/rest/AbstractRestTestPlugin.java | 104 ++++++++++++++++++ .../gradle/test/rest/JavaRestTestPlugin.java | 69 ++++++++++++ .../gradle/test/rest/YamlRestTestPlugin.java | 67 ++--------- .../elasticsearch.java-rest-test.properties | 20 ++++ modules/aggs-matrix-stats/build.gradle | 2 - modules/analysis-common/build.gradle | 1 + .../common/QueryStringWithAnalyzersIT.java | 0 modules/geo/build.gradle | 2 - modules/ingest-common/build.gradle | 1 + .../ingest/common/IngestRestartIT.java | 0 modules/ingest-geoip/build.gradle | 2 - modules/ingest-user-agent/build.gradle | 1 - modules/kibana/build.gradle | 4 +- .../kibana/KibanaSystemIndexIT.java | 0 modules/lang-mustache/build.gradle | 2 + .../mustache/MultiSearchTemplateIT.java | 0 .../script/mustache/SearchTemplateIT.java | 0 .../SearchTemplateWithoutContentIT.java | 0 modules/mapper-extras/build.gradle | 1 + .../TokenCountFieldMapperIntegrationIT.java | 4 +- modules/parent-join/build.gradle | 1 + 22 files changed, 221 insertions(+), 66 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/AbstractRestTestPlugin.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java create mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java-rest-test.properties rename modules/analysis-common/src/{test => internalClusterTest}/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java (100%) rename modules/ingest-common/src/{test => internalClusterTest}/java/org/elasticsearch/ingest/common/IngestRestartIT.java (100%) rename modules/kibana/src/{test => javaRestTest}/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java (100%) rename modules/lang-mustache/src/{test => internalClusterTest}/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java (100%) rename modules/lang-mustache/src/{test => internalClusterTest}/java/org/elasticsearch/script/mustache/SearchTemplateIT.java (100%) rename modules/lang-mustache/src/{test => javaRestTest}/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java (100%) rename modules/mapper-extras/src/{test => javaRestTest}/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java (98%) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index e775f64c81050..d9f6464045d38 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -115,6 +115,12 @@ class PluginBuildPlugin implements Plugin { if (isModule == false || isXPackModule) { addNoticeGeneration(project, extension1) } +// if(project.pluginManager.hasPlugin("elasticsearch.yaml-rest-test") +// || project.pluginManager.hasPlugin("elasticsearch.java-rest-test")) { +// //disable integTest task if project has been converted to use yaml or java rest test plugin +// project.tasks.integTest.enabled = false +// } + } project.tasks.named('testingConventions').configure { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/AbstractRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/AbstractRestTestPlugin.java new file mode 100644 index 0000000000000..5150b601850ce --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/AbstractRestTestPlugin.java @@ -0,0 +1,104 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test.rest; + +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; +import org.elasticsearch.gradle.test.RestIntegTestTask; +import org.elasticsearch.gradle.testclusters.RestTestRunnerTask; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.plugins.JavaBasePlugin; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.bundling.Zip; + +/** + * Helper parent to configure the necessary tasks and dependencies. + */ +public abstract class AbstractRestTestPlugin implements Plugin { + + /** + * Creates a task with the source set name of type {@link RestIntegTestTask} + */ + protected RestIntegTestTask setupTask(Project project, String sourceSetName) { + // create task - note can not use .register due to the work in RestIntegTestTask's constructor :( + // see: https://github.com/elastic/elasticsearch/issues/47804 + RestIntegTestTask testTask = project.getTasks().create(sourceSetName, RestIntegTestTask.class); + testTask.setGroup(JavaBasePlugin.VERIFICATION_GROUP); + testTask.setDescription("Runs the REST tests against an external cluster"); + // make the new test run after unit tests + testTask.mustRunAfter(project.getTasks().named("test")); + return testTask; + } + + /** + * Creates the runner task and configures the test clusters + */ + protected RestTestRunnerTask setupRunnerTask(Project project, RestIntegTestTask testTask, SourceSet sourceSet) { + RestTestRunnerTask runner = testTask.getRunner(); + runner.setTestClassesDirs(sourceSet.getOutput().getClassesDirs()); + runner.setClasspath(sourceSet.getRuntimeClasspath()); + + // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster + project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> { + Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin"); + testTask.dependsOn(bundle); + if (project.getPath().startsWith(":modules:")) { + runner.getClusters().forEach(c -> c.module(bundle.getArchiveFile())); + } else { + runner.getClusters().forEach(c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))); + } + }); + + // es-plugins may declare dependencies on additional modules, add those to the test cluster too. + project.afterEvaluate(p -> { + PluginPropertiesExtension pluginPropertiesExtension = project.getExtensions().findByType(PluginPropertiesExtension.class); + if (pluginPropertiesExtension != null) { // not all projects are defined as plugins + pluginPropertiesExtension.getExtendedPlugins().forEach(pluginName -> { + Project extensionProject = project.getProject().findProject(":modules:" + pluginName); + if (extensionProject != null) { // extension plugin may be defined, but not required to be a module + Zip extensionBundle = (Zip) extensionProject.getTasks().getByName("bundlePlugin"); + testTask.dependsOn(extensionBundle); + runner.getClusters().forEach(c -> c.module(extensionBundle.getArchiveFile())); + } + }); + } + }); + return runner; + } + + /** + * Setup the dependencies needed for the REST tests. + */ + protected void setupDependencies(Project project, SourceSet sourceSet) { + if (BuildParams.isInternal()) { + project.getDependencies().add(sourceSet.getImplementationConfigurationName(), project.project(":test:framework")); + } else { + project.getDependencies() + .add( + sourceSet.getImplementationConfigurationName(), + "org.elasticsearch.test:framework:" + VersionProperties.getElasticsearch() + ); + } + + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java new file mode 100644 index 0000000000000..6bdd7cf326b38 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test.rest; + +import org.elasticsearch.gradle.ElasticsearchJavaPlugin; +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; +import org.elasticsearch.gradle.test.RestIntegTestTask; +import org.elasticsearch.gradle.testclusters.RestTestRunnerTask; +import org.elasticsearch.gradle.testclusters.TestClustersPlugin; +import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.plugins.JavaBasePlugin; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.SourceSetContainer; +import org.gradle.api.tasks.bundling.Zip; + +/** + * Apply this plugin to run the Java based REST tests. + */ +public class JavaRestTestPlugin extends AbstractRestTestPlugin { + + public static final String SOURCE_SET_NAME = "javaRestTest"; + + @Override + public void apply(Project project) { + + project.getPluginManager().apply(ElasticsearchJavaPlugin.class); + project.getPluginManager().apply(TestClustersPlugin.class); + + // create source set + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet javaTestSourceSet = sourceSets.create(SOURCE_SET_NAME); + + // setup the javaRestTest task + RestIntegTestTask javaRestTestTask = setupTask(project, SOURCE_SET_NAME); + + // setup the runner task + setupRunnerTask(project, javaRestTestTask, javaTestSourceSet); + + // setup dependencies + setupDependencies(project, javaTestSourceSet); + + // setup IDE + GradleUtils.setupIdeForTestSourceSet(project, javaTestSourceSet); + + // wire this task into check + project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure(check -> check.dependsOn(javaRestTestTask)); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java index 676d2ffbf6268..47be0809cf2bb 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java @@ -37,84 +37,37 @@ /** * Apply this plugin to run the YAML based REST tests. */ -public class YamlRestTestPlugin implements Plugin { +public class YamlRestTestPlugin extends AbstractRestTestPlugin { public static final String SOURCE_SET_NAME = "yamlRestTest"; @Override public void apply(Project project) { - // yaml Rest tests require a Java test runner project.getPluginManager().apply(ElasticsearchJavaPlugin.class); - // to spin up the external cluster project.getPluginManager().apply(TestClustersPlugin.class); - // to copy around the yaml tests and json spec project.getPluginManager().apply(RestResourcesPlugin.class); - // note - source sets are not created via org.elasticsearch.gradle.util.GradleUtils.addTestSourceSet since unlike normal tests - // we only want the yamlRestTestSourceSet on the classpath by default. The yaml tests should be pure black box testing over HTTP and - // such it should not need the main class on the class path. Also, there are some special setup steps unique to YAML REST tests. - // create source set SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); SourceSet yamlTestSourceSet = sourceSets.create(SOURCE_SET_NAME); - // create task - note can not use .register due to the work in RestIntegTestTask's constructor :( - // see: https://github.com/elastic/elasticsearch/issues/47804 - RestIntegTestTask yamlRestTestTask = project.getTasks().create(SOURCE_SET_NAME, RestIntegTestTask.class); - yamlRestTestTask.setGroup(JavaBasePlugin.VERIFICATION_GROUP); - yamlRestTestTask.setDescription("Runs the YAML based REST tests against an external cluster"); + // setup the yamlRestTest task + RestIntegTestTask yamlRestTestTask = setupTask(project, SOURCE_SET_NAME); + + // setup the runner task + setupRunnerTask(project, yamlRestTestTask, yamlTestSourceSet); - // setup task dependency - if (BuildParams.isInternal()) { - project.getDependencies().add(yamlTestSourceSet.getImplementationConfigurationName(), project.project(":test:framework")); - } else { - project.getDependencies() - .add( - yamlTestSourceSet.getImplementationConfigurationName(), - "org.elasticsearch.test:framework:" + VersionProperties.getElasticsearch() - ); - } + // setup the dependencies + setupDependencies(project, yamlTestSourceSet); // setup the copy for the rest resources project.getTasks().withType(CopyRestApiTask.class, copyRestApiTask -> { copyRestApiTask.sourceSetName = SOURCE_SET_NAME; project.getTasks().named(yamlTestSourceSet.getProcessResourcesTaskName()).configure(t -> t.dependsOn(copyRestApiTask)); }); - project.getTasks().withType(CopyRestTestsTask.class, copyRestTestTask -> { copyRestTestTask.sourceSetName = SOURCE_SET_NAME; }); - - // make the new test run after unit tests - yamlRestTestTask.mustRunAfter(project.getTasks().named("test")); - - // setup the runner - RestTestRunnerTask runner = yamlRestTestTask.getRunner(); - runner.setTestClassesDirs(yamlTestSourceSet.getOutput().getClassesDirs()); - runner.setClasspath(yamlTestSourceSet.getRuntimeClasspath()); - - // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster - project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> { - Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin"); - yamlRestTestTask.dependsOn(bundle); - if (project.getPath().startsWith(":modules:")) { - runner.getClusters().forEach(c -> c.module(bundle.getArchiveFile())); - } else { - runner.getClusters().forEach(c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))); - } - }); - - // es-plugins may declare dependencies on additional modules, add those to the test cluster too. - project.afterEvaluate(p -> { - PluginPropertiesExtension pluginPropertiesExtension = project.getExtensions().findByType(PluginPropertiesExtension.class); - if (pluginPropertiesExtension != null) { // not all projects are defined as plugins - pluginPropertiesExtension.getExtendedPlugins().forEach(pluginName -> { - Project extensionProject = project.getProject().findProject(":modules:" + pluginName); - if (extensionProject != null) { // extension plugin may be defined, but not required to be a module - Zip extensionBundle = (Zip) extensionProject.getTasks().getByName("bundlePlugin"); - yamlRestTestTask.dependsOn(extensionBundle); - runner.getClusters().forEach(c -> c.module(extensionBundle.getArchiveFile())); - } - }); - } + project.getTasks().withType(CopyRestTestsTask.class, copyRestTestTask -> { + copyRestTestTask.sourceSetName = SOURCE_SET_NAME; }); // setup IDE diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java-rest-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java-rest-test.properties new file mode 100644 index 0000000000000..6eda5adec7b83 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java-rest-test.properties @@ -0,0 +1,20 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +implementation-class=org.elasticsearch.gradle.test.rest.JavaRestTestPlugin diff --git a/modules/aggs-matrix-stats/build.gradle b/modules/aggs-matrix-stats/build.gradle index aad781fb2cacd..e01fe1c4ad959 100644 --- a/modules/aggs-matrix-stats/build.gradle +++ b/modules/aggs-matrix-stats/build.gradle @@ -28,5 +28,3 @@ restResources { includeCore '_common', 'indices', 'cluster', 'index', 'search', 'nodes' } } - -integTest.enabled = false diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle index f631aa725d017..bce1057214526 100644 --- a/modules/analysis-common/build.gradle +++ b/modules/analysis-common/build.gradle @@ -17,6 +17,7 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'Adds "built in" analyzers to Elasticsearch.' diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java similarity index 100% rename from modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java rename to modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java diff --git a/modules/geo/build.gradle b/modules/geo/build.gradle index c089b395df062..5381d0e4a0451 100644 --- a/modules/geo/build.gradle +++ b/modules/geo/build.gradle @@ -31,6 +31,4 @@ restResources { artifacts { restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test')) } - -integTest.enabled = false test.enabled = false diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 2edf77c0c972c..4b4fa8e81ef62 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -17,6 +17,7 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources' diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java similarity index 100% rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java rename to modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index f0525a05d6f9d..ede1705be5a22 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -43,8 +43,6 @@ restResources { } } -integTest.enabled = false - task copyDefaultGeoIp2DatabaseFiles(type: Copy) { from { zipTree(configurations.testCompileClasspath.files.find { it.name.contains('geolite2-databases') }) } into "${project.buildDir}/ingest-geoip" diff --git a/modules/ingest-user-agent/build.gradle b/modules/ingest-user-agent/build.gradle index 3d38ea9999574..6870c26032e86 100644 --- a/modules/ingest-user-agent/build.gradle +++ b/modules/ingest-user-agent/build.gradle @@ -33,4 +33,3 @@ testClusters.all { extraConfigFile 'ingest-user-agent/test-regexes.yml', file('src/test/test-regexes.yml') } -integTest.enabled = false diff --git a/modules/kibana/build.gradle b/modules/kibana/build.gradle index 8c102080edc4a..e432c328574e7 100644 --- a/modules/kibana/build.gradle +++ b/modules/kibana/build.gradle @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +apply plugin: 'elasticsearch.java-rest-test' esplugin { description 'Plugin exposing APIs for Kibana system indices' @@ -26,6 +27,7 @@ dependencies { api project(path: ':modules:reindex') } -testClusters.integTest { +testClusters.all { module file(project(':modules:reindex').tasks.bundlePlugin.archiveFile) } + diff --git a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java b/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java similarity index 100% rename from modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java rename to modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle index afa582bec966a..c3fae15d56456 100644 --- a/modules/lang-mustache/build.gradle +++ b/modules/lang-mustache/build.gradle @@ -17,6 +17,8 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.java-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'Mustache scripting integration for Elasticsearch' diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java similarity index 100% rename from modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java rename to modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java similarity index 100% rename from modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java rename to modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java b/modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java similarity index 100% rename from modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java rename to modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java diff --git a/modules/mapper-extras/build.gradle b/modules/mapper-extras/build.gradle index fc60820b33398..8c7e09766e63e 100644 --- a/modules/mapper-extras/build.gradle +++ b/modules/mapper-extras/build.gradle @@ -17,6 +17,7 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.java-rest-test' esplugin { description 'Adds advanced field mappers' diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java b/modules/mapper-extras/src/javaRestTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java similarity index 98% rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java rename to modules/mapper-extras/src/javaRestTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java index 9c5c3b76523f8..0f82cb0ef39cf 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java +++ b/modules/mapper-extras/src/javaRestTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -34,6 +34,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.rest.ESRestTestCase; import java.io.IOException; import java.util.ArrayList; @@ -46,6 +47,7 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +//TODO: convert to use ESRestTestCase instead of ESIntegTestCase public class TokenCountFieldMapperIntegrationIT extends ESIntegTestCase { @ParametersFactory public static Iterable buildParameters() { diff --git a/modules/parent-join/build.gradle b/modules/parent-join/build.gradle index d5ce578c8189f..6cb0ad10f9901 100644 --- a/modules/parent-join/build.gradle +++ b/modules/parent-join/build.gradle @@ -17,6 +17,7 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'This module adds the support parent-child queries and aggregations' From 45bf3d85a0756b528cb26e04c01d41766a0b5b71 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 20 Jul 2020 18:33:13 -0500 Subject: [PATCH 2/5] should all work --- .../gradle/plugin/PluginBuildPlugin.groovy | 11 ++++---- .../TokenCountFieldMapperIntegrationIT.java | 3 -- .../AbstractParentChildTestCase.java | 0 .../join/aggregations/ChildrenIT.java | 0 .../join/aggregations/ParentIT.java | 0 .../join/query/ChildQuerySearchIT.java | 0 .../elasticsearch/join/query/InnerHitsIT.java | 0 .../join/query/ParentChildTestCase.java | 0 modules/percolator/build.gradle | 15 +--------- .../percolator/PercolatorQuerySearchIT.java | 0 modules/rank-eval/build.gradle | 1 + .../index/rankeval/RankEvalRequestIT.java | 0 modules/reindex/build.gradle | 8 ++++-- .../documentation/ReindexDocumentationIT.java | 0 .../index/reindex/ManyDocumentsIT.java | 2 +- .../reindex/ReindexWithoutContentIT.java | 2 +- .../remote/ReindexFromOldRemoteIT.java | 2 +- modules/repository-url/build.gradle | 2 -- modules/transport-netty4/build.gradle | 28 +++++++++++++++---- .../elasticsearch/ESNetty4IntegTestCase.java | 0 .../netty4/Netty4HttpRequestSizeLimitIT.java | 4 +-- .../http/netty4/Netty4PipeliningIT.java | 0 .../rest/discovery/Zen2RestApiIT.java | 0 .../transport/netty4/ESLoggingHandlerIT.java | 0 ...Netty4TransportMultiPortIntegrationIT.java | 0 .../Netty4TransportPublishAddressIT.java | 0 .../rest/Netty4BadRequestIT.java | 0 .../rest/Netty4HeadBodyIsEmptyIT.java | 0 28 files changed, 40 insertions(+), 38 deletions(-) rename modules/parent-join/src/{test => internalClusterTest}/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java (100%) rename modules/parent-join/src/{test => internalClusterTest}/java/org/elasticsearch/join/aggregations/ChildrenIT.java (100%) rename modules/parent-join/src/{test => internalClusterTest}/java/org/elasticsearch/join/aggregations/ParentIT.java (100%) rename modules/parent-join/src/{test => internalClusterTest}/java/org/elasticsearch/join/query/ChildQuerySearchIT.java (100%) rename modules/parent-join/src/{test => internalClusterTest}/java/org/elasticsearch/join/query/InnerHitsIT.java (100%) rename modules/parent-join/src/{test => internalClusterTest}/java/org/elasticsearch/join/query/ParentChildTestCase.java (100%) rename modules/percolator/src/{test => internalClusterTest}/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java (100%) rename modules/rank-eval/src/{test => internalClusterTest}/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java (100%) rename modules/reindex/src/{test => internalClusterTest}/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java (100%) rename modules/reindex/src/{test => javaRestTest}/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java (99%) rename modules/reindex/src/{test => javaRestTest}/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java (96%) rename modules/reindex/src/{test => javaRestTest}/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java (99%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/ESNetty4IntegTestCase.java (100%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java (99%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java (100%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java (100%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java (100%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java (100%) rename modules/transport-netty4/src/{test => internalClusterTest}/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java (100%) rename modules/transport-netty4/src/{test => javaRestTest}/java/org/elasticsearch/rest/Netty4BadRequestIT.java (100%) rename modules/transport-netty4/src/{test => javaRestTest}/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java (100%) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index d9f6464045d38..284464520bd75 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -115,12 +115,11 @@ class PluginBuildPlugin implements Plugin { if (isModule == false || isXPackModule) { addNoticeGeneration(project, extension1) } -// if(project.pluginManager.hasPlugin("elasticsearch.yaml-rest-test") -// || project.pluginManager.hasPlugin("elasticsearch.java-rest-test")) { -// //disable integTest task if project has been converted to use yaml or java rest test plugin -// project.tasks.integTest.enabled = false -// } - + if(project.pluginManager.hasPlugin("elasticsearch.yaml-rest-test") + || project.pluginManager.hasPlugin("elasticsearch.java-rest-test")) { + //disable integTest task if project has been converted to use yaml or java rest test plugin + project.tasks.integTest.enabled = false + } } project.tasks.named('testingConventions').configure { diff --git a/modules/mapper-extras/src/javaRestTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java b/modules/mapper-extras/src/javaRestTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java index 0f82cb0ef39cf..0113cfa4c32d6 100644 --- a/modules/mapper-extras/src/javaRestTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java +++ b/modules/mapper-extras/src/javaRestTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java @@ -21,7 +21,6 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.bulk.BulkResponse; @@ -34,7 +33,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.rest.ESRestTestCase; import java.io.IOException; import java.util.ArrayList; @@ -47,7 +45,6 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; -//TODO: convert to use ESRestTestCase instead of ESIntegTestCase public class TokenCountFieldMapperIntegrationIT extends ESIntegTestCase { @ParametersFactory public static Iterable buildParameters() { diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java rename to modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java rename to modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ParentIT.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentIT.java rename to modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ParentIT.java diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ChildQuerySearchIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/query/ChildQuerySearchIT.java rename to modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java rename to modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ParentChildTestCase.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java rename to modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ParentChildTestCase.java diff --git a/modules/percolator/build.gradle b/modules/percolator/build.gradle index ce1e68025a044..cbc6aebf87de2 100644 --- a/modules/percolator/build.gradle +++ b/modules/percolator/build.gradle @@ -17,6 +17,7 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'Percolator module adds capability to index queries and query these queries by specifying documents' @@ -28,22 +29,8 @@ dependencies { testImplementation project(':modules:geo') } -tasks.named('integTestRunner').configure { - exclude '**/PercolatorQuerySearchIT.class' -} - -tasks.register('internalClusterTest', Test) { - include '**/PercolatorQuerySearchIT.class' -} - -tasks.named('check').configure { - dependsOn 'internalClusterTest' -} - restResources { restApi { includeCore '_common', 'indices', 'index', 'search', 'msearch' } } - -integTest.enabled = false diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java similarity index 100% rename from modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java rename to modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle index 2bf54796abf2d..135295307fc1f 100644 --- a/modules/rank-eval/build.gradle +++ b/modules/rank-eval/build.gradle @@ -17,6 +17,7 @@ * under the License. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'The Rank Eval module adds APIs to evaluate ranking quality.' diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java similarity index 100% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java rename to modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 3d5d745255293..3c74bb99f404a 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -25,6 +25,8 @@ import org.elasticsearch.gradle.info.BuildParams apply plugin: 'elasticsearch.test-with-dependencies' apply plugin: 'elasticsearch.jdk-download' apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.java-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'The Reindex module adds APIs to reindex from one index to another or update documents in place.' @@ -107,12 +109,12 @@ jdks { if (Os.isFamily(Os.FAMILY_WINDOWS)) { logger.warn("Disabling reindex-from-old tests because we can't get the pid file on windows") - integTest.runner { + javaRestTest.runner { systemProperty "tests.fromOld", "false" } } else if (rootProject.rootDir.toString().contains(" ")) { logger.warn("Disabling reindex-from-old tests because Elasticsearch 1.7 won't start with spaces in the path") - integTest.runner { + javaRestTest.runner { systemProperty "tests.fromOld", "false" } } else { @@ -153,7 +155,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { } } - integTest { + javaRestTest { dependsOn fixture runner { systemProperty "tests.fromOld", "true" diff --git a/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java similarity index 100% rename from modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java rename to modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java similarity index 99% rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java rename to modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java index fd8cf49cc39ad..6eed35200808c 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java +++ b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java similarity index 96% rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java rename to modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java index 73745ca690d74..683f02b6f967d 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java +++ b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java similarity index 99% rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java rename to modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java index cc49d28471aee..407b081abbeec 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java +++ b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle index 6f2dff252d9cc..1d79fce831d74 100644 --- a/modules/repository-url/build.gradle +++ b/modules/repository-url/build.gradle @@ -36,8 +36,6 @@ restResources { } } -integTest.enabled = false - // This directory is shared between two URL repositories and one FS repository in YAML integration tests File repositoryDir = new File(project.buildDir, "shared-repository") diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index d91ccd9c1bdbf..cb2cb0fe462b2 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -20,8 +20,12 @@ import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.test.RestIntegTestTask +import org.elasticsearch.gradle.test.rest.JavaRestTestPlugin +import org.elasticsearch.gradle.test.InternalClusterTestPlugin apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.java-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' /* TODOs: @@ -55,7 +59,7 @@ tasks.named("dependencyLicenses").configure { mapping from: /netty-.*/, to: 'netty' } -test { +internalClusterTest { /* * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each * other if we allow them to set the number of available processors as it's set-once in Netty. @@ -63,7 +67,7 @@ test { systemProperty 'es.set.netty.runtime.available.processors', 'false' } -integTestRunner { +javaRestTestRunner { /* * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each * other if we allow them to set the number of available processors as it's set-once in Netty. @@ -77,15 +81,29 @@ TaskProvider pooledTest = tasks.register("pooledTest", Test) { systemProperty 'es.use_unpooled_allocator', 'false' } -RestIntegTestTask pooledIntegTest = tasks.create("pooledIntegTest", RestIntegTestTask) { +TaskProvider pooledInternalClusterTest = tasks.register("pooledInternalClusterTest", Test) { + include '**/*IT.class' + systemProperty 'es.set.netty.runtime.available.processors', 'false' + systemProperty 'es.use_unpooled_allocator', 'false' + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs()) + setClasspath(internalTestSourceSet.getRuntimeClasspath()) +} + +RestIntegTestTask pooledJavaRestTest = tasks.create("pooledJavaRestTest", RestIntegTestTask) { runner { systemProperty 'es.set.netty.runtime.available.processors', 'false' + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet javaRestTestSourceSet = sourceSets.getByName(JavaRestTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(javaRestTestSourceSet.getOutput().getClassesDirs()) + setClasspath(javaRestTestSourceSet.getRuntimeClasspath()) } } -testClusters.pooledIntegTest { +testClusters.pooledJavaRestTest { systemProperty 'es.use_unpooled_allocator', 'false' } -check.dependsOn(pooledTest, pooledIntegTest) +check.dependsOn(pooledTest, pooledJavaRestTest, pooledInternalClusterTest) thirdPartyAudit { ignoreMissingClasses( diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/ESNetty4IntegTestCase.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/ESNetty4IntegTestCase.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java similarity index 99% rename from modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java index f011e9a8abb27..d9716fb505ad3 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -20,6 +20,7 @@ package org.elasticsearch.http.netty4; import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.util.ReferenceCounted; import org.elasticsearch.ESNetty4IntegTestCase; import org.elasticsearch.common.collect.Tuple; @@ -31,7 +32,6 @@ import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import io.netty.handler.codec.http.HttpResponseStatus; import java.util.ArrayList; import java.util.Collection; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java rename to modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java rename to modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java rename to modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java From 319d6fac4b71d76dccd37fa44e73f0176f7aa70f Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Tue, 21 Jul 2020 07:53:13 -0500 Subject: [PATCH 3/5] spotless --- .../gradle/test/rest/JavaRestTestPlugin.java | 6 ------ .../gradle/test/rest/YamlRestTestPlugin.java | 10 +--------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java index 6bdd7cf326b38..06c2ffd47c2fc 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java @@ -20,19 +20,13 @@ package org.elasticsearch.gradle.test.rest; import org.elasticsearch.gradle.ElasticsearchJavaPlugin; -import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; import org.elasticsearch.gradle.test.RestIntegTestTask; -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.elasticsearch.gradle.util.GradleUtils; -import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.SourceSetContainer; -import org.gradle.api.tasks.bundling.Zip; /** * Apply this plugin to run the Java based REST tests. diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java index 47be0809cf2bb..15e0faedcfed6 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java @@ -20,19 +20,13 @@ package org.elasticsearch.gradle.test.rest; import org.elasticsearch.gradle.ElasticsearchJavaPlugin; -import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; import org.elasticsearch.gradle.test.RestIntegTestTask; -import org.elasticsearch.gradle.testclusters.RestTestRunnerTask; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.elasticsearch.gradle.util.GradleUtils; -import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.SourceSetContainer; -import org.gradle.api.tasks.bundling.Zip; /** * Apply this plugin to run the YAML based REST tests. @@ -66,9 +60,7 @@ public void apply(Project project) { copyRestApiTask.sourceSetName = SOURCE_SET_NAME; project.getTasks().named(yamlTestSourceSet.getProcessResourcesTaskName()).configure(t -> t.dependsOn(copyRestApiTask)); }); - project.getTasks().withType(CopyRestTestsTask.class, copyRestTestTask -> { - copyRestTestTask.sourceSetName = SOURCE_SET_NAME; - }); + project.getTasks().withType(CopyRestTestsTask.class, copyRestTestTask -> { copyRestTestTask.sourceSetName = SOURCE_SET_NAME; }); // setup IDE GradleUtils.setupIdeForTestSourceSet(project, yamlTestSourceSet); From d5e45728ec18fa707685582cc0db065d45591f59 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Tue, 21 Jul 2020 14:08:14 -0500 Subject: [PATCH 4/5] minor review changes --- .../gradle/plugin/PluginBuildPlugin.groovy | 2 +- modules/transport-netty4/build.gradle | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index 284464520bd75..3d983358fcef4 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -115,7 +115,7 @@ class PluginBuildPlugin implements Plugin { if (isModule == false || isXPackModule) { addNoticeGeneration(project, extension1) } - if(project.pluginManager.hasPlugin("elasticsearch.yaml-rest-test") + if (project.pluginManager.hasPlugin("elasticsearch.yaml-rest-test") || project.pluginManager.hasPlugin("elasticsearch.java-rest-test")) { //disable integTest task if project has been converted to use yaml or java rest test plugin project.tasks.integTest.enabled = false diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index cb2cb0fe462b2..95a3ef9d37a49 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -59,7 +59,7 @@ tasks.named("dependencyLicenses").configure { mapping from: /netty-.*/, to: 'netty' } -internalClusterTest { +test { /* * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each * other if we allow them to set the number of available processors as it's set-once in Netty. @@ -67,11 +67,11 @@ internalClusterTest { systemProperty 'es.set.netty.runtime.available.processors', 'false' } +internalClusterTest { + systemProperty 'es.set.netty.runtime.available.processors', 'false' +} + javaRestTestRunner { - /* - * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each - * other if we allow them to set the number of available processors as it's set-once in Netty. - */ systemProperty 'es.set.netty.runtime.available.processors', 'false' } From 01521f972fc1d79ca201ce7e3f14ef1444f24626 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Tue, 21 Jul 2020 16:09:59 -0500 Subject: [PATCH 5/5] withPlugin and static methods --- .../gradle/plugin/PluginBuildPlugin.groovy | 13 ++++++++----- .../gradle/test/rest/JavaRestTestPlugin.java | 7 ++++++- ...bstractRestTestPlugin.java => RestTestUtil.java} | 13 +++++++------ .../gradle/test/rest/YamlRestTestPlugin.java | 7 ++++++- 4 files changed, 27 insertions(+), 13 deletions(-) rename buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/{AbstractRestTestPlugin.java => RestTestUtil.java} (90%) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index 3d983358fcef4..b108ed3f99d14 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -115,11 +115,14 @@ class PluginBuildPlugin implements Plugin { if (isModule == false || isXPackModule) { addNoticeGeneration(project, extension1) } - if (project.pluginManager.hasPlugin("elasticsearch.yaml-rest-test") - || project.pluginManager.hasPlugin("elasticsearch.java-rest-test")) { - //disable integTest task if project has been converted to use yaml or java rest test plugin - project.tasks.integTest.enabled = false - } + } + + //disable integTest task if project has been converted to use yaml or java rest test plugin + project.pluginManager.withPlugin("elasticsearch.yaml-rest-test") { + project.tasks.integTest.enabled = false + } + project.pluginManager.withPlugin("elasticsearch.java-rest-test") { + project.tasks.integTest.enabled = false } project.tasks.named('testingConventions').configure { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java index 06c2ffd47c2fc..4738d3525cf81 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java @@ -23,15 +23,20 @@ import org.elasticsearch.gradle.test.RestIntegTestTask; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.SourceSetContainer; +import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupDependencies; +import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupRunnerTask; +import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupTask; + /** * Apply this plugin to run the Java based REST tests. */ -public class JavaRestTestPlugin extends AbstractRestTestPlugin { +public class JavaRestTestPlugin implements Plugin { public static final String SOURCE_SET_NAME = "javaRestTest"; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/AbstractRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java similarity index 90% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/AbstractRestTestPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java index 5150b601850ce..ed3711183b401 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/AbstractRestTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java @@ -24,21 +24,22 @@ import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; import org.elasticsearch.gradle.test.RestIntegTestTask; import org.elasticsearch.gradle.testclusters.RestTestRunnerTask; -import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.bundling.Zip; /** - * Helper parent to configure the necessary tasks and dependencies. + * Utility class to configure the necessary tasks and dependencies. */ -public abstract class AbstractRestTestPlugin implements Plugin { +public class RestTestUtil { + + private RestTestUtil() {} /** * Creates a task with the source set name of type {@link RestIntegTestTask} */ - protected RestIntegTestTask setupTask(Project project, String sourceSetName) { + static RestIntegTestTask setupTask(Project project, String sourceSetName) { // create task - note can not use .register due to the work in RestIntegTestTask's constructor :( // see: https://github.com/elastic/elasticsearch/issues/47804 RestIntegTestTask testTask = project.getTasks().create(sourceSetName, RestIntegTestTask.class); @@ -52,7 +53,7 @@ protected RestIntegTestTask setupTask(Project project, String sourceSetName) { /** * Creates the runner task and configures the test clusters */ - protected RestTestRunnerTask setupRunnerTask(Project project, RestIntegTestTask testTask, SourceSet sourceSet) { + static RestTestRunnerTask setupRunnerTask(Project project, RestIntegTestTask testTask, SourceSet sourceSet) { RestTestRunnerTask runner = testTask.getRunner(); runner.setTestClassesDirs(sourceSet.getOutput().getClassesDirs()); runner.setClasspath(sourceSet.getRuntimeClasspath()); @@ -88,7 +89,7 @@ protected RestTestRunnerTask setupRunnerTask(Project project, RestIntegTestTask /** * Setup the dependencies needed for the REST tests. */ - protected void setupDependencies(Project project, SourceSet sourceSet) { + static void setupDependencies(Project project, SourceSet sourceSet) { if (BuildParams.isInternal()) { project.getDependencies().add(sourceSet.getImplementationConfigurationName(), project.project(":test:framework")); } else { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java index 15e0faedcfed6..45884b036703e 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java @@ -23,15 +23,20 @@ import org.elasticsearch.gradle.test.RestIntegTestTask; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.SourceSetContainer; +import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupDependencies; +import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupRunnerTask; +import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupTask; + /** * Apply this plugin to run the YAML based REST tests. */ -public class YamlRestTestPlugin extends AbstractRestTestPlugin { +public class YamlRestTestPlugin implements Plugin { public static final String SOURCE_SET_NAME = "yamlRestTest";