From c7d36e063ed3f789aaaa00ba3e5d351fb46d6034 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 5 Feb 2020 17:57:29 -0600 Subject: [PATCH 01/23] Smarter copying of the rest specs and tests This PR addresses the uncessary copying of the rest specs and allows for better semantics for which specs and tests are copied. By default the rest specs will get copied if the project applies `elasticsearch.standalone-rest-test` or `esplugin` and the project has rest tests. If the project does not have rest tests, you can configure the project to copy the specs anyway with `restApiSpec { alwaysCopySpec true }`. This PR also removes the need for dozens of places where the x-pack specs were copied by supporting copying of the x-pack rest specs if the project starts with `:x-pack` and the preceeding applies. The plugin introduced here can also copy the rest tests to the local project through similar configuration. The new plugin allows a user to minimize the surface area of which rest specs are copyied. Per project can be configured to include only a subset of the specs (or tests) `restApiSpec { includeXpackSpec "ccr" }` through a custom plugin extention. Only copying the specs when actually needed and only copying the minimal set of specs should help with build cache hit rates since we can define only what is in use. Project level optimizations for build cache hit rates are not included with this PR. Also, with this PR you can no longer use the includePackaged flag on integTest task, instead you will need to configure the plugin extention `restApiSpec { copyCoreTests true }`. Support for copying the rest tests from an external JAR is also dropped. While techincally non-passive, I can not imaging why a plugin developer (or any user of that JAR) would need the rest tests accessible from our `integTest` task. (the tests are still aviable in jar) The following items are included in this PR: `apply plugin: 'elasticsearch.rest-api-spec-for-testing'` (applied by default to `elasticsearch.standalone-rest-test` and `esplugin`) ``` restApiSpec { includeCoreSpec "foo" //will include the core specs that start with foo includeXpackSpec "bar" //will include x-pack specs that start with bar copyCoreTests true //will copy the core rest tests to the local project includeCoreTests "foo" //will include the core tests that start with foo copyXpackTests true //will copy the x-pack rest tests to the local project includeXpackTests "bar" //will include the x-pack tests that start with bar alwaysCopySpec true //will always copy the specs (core and/or x-pack), even if no tests exist to to the local project } ``` --- .../gradle/plugin/PluginBuildPlugin.groovy | 2 + .../gradle/test/RestIntegTestTask.groovy | 55 +------ .../test/StandaloneRestTestPlugin.groovy | 2 + .../precommit/TestingConventionsTasks.java | 2 +- .../gradle/test/RestApiSpecExtension.java | 70 +++++++++ .../test/RestApiSpecForTestingPlugin.java | 141 ++++++++++++++++++ ...earch.rest-api-spec-for-testing.properties | 20 +++ .../gradle/info/BuildParams.java | 2 +- client/rest-high-level/build.gradle | 21 +-- distribution/archives/build.gradle | 5 +- distribution/docker/build.gradle | 6 - docs/build.gradle | 4 + qa/mixed-cluster/build.gradle | 13 +- qa/remote-clusters/build.gradle | 9 -- qa/rolling-upgrade/build.gradle | 15 -- qa/smoke-test-multinode/build.gradle | 4 +- rest-api-spec/build.gradle | 10 ++ x-pack/docs/build.gradle | 8 - x-pack/plugin/build.gradle | 7 + x-pack/plugin/ccr/qa/build.gradle | 12 -- x-pack/plugin/ccr/qa/rest/build.gradle | 4 + x-pack/plugin/enrich/qa/build.gradle | 9 -- x-pack/plugin/enrich/qa/rest/build.gradle | 4 + x-pack/plugin/eql/qa/build.gradle | 9 -- x-pack/plugin/eql/qa/rest/build.gradle | 4 + x-pack/plugin/graph/qa/build.gradle | 17 --- .../graph/qa/with-security/build.gradle | 4 + x-pack/plugin/ilm/qa/build.gradle | 11 -- x-pack/plugin/ilm/qa/rest/build.gradle | 5 + x-pack/plugin/ml/qa/build.gradle | 17 --- .../ml/qa/ml-with-security/build.gradle | 11 +- x-pack/plugin/security/qa/build.gradle | 11 -- x-pack/qa/build.gradle | 16 -- .../build.gradle | 5 +- x-pack/qa/full-cluster-restart/build.gradle | 20 --- x-pack/qa/rolling-upgrade-basic/build.gradle | 20 --- .../build.gradle | 19 --- x-pack/qa/rolling-upgrade/build.gradle | 19 --- 38 files changed, 300 insertions(+), 313 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecExtension.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecForTestingPlugin.java create mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-spec-for-testing.properties diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index 52f225f11794a..58407399fdb46 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -24,6 +24,7 @@ import org.elasticsearch.gradle.NoticeTask import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.test.RestApiSpecForTestingPlugin import org.elasticsearch.gradle.test.RestIntegTestTask import org.elasticsearch.gradle.testclusters.RunTask import org.elasticsearch.gradle.testclusters.TestClustersPlugin @@ -51,6 +52,7 @@ class PluginBuildPlugin implements Plugin { void apply(Project project) { project.pluginManager.apply(BuildPlugin) project.pluginManager.apply(TestClustersPlugin) + project.pluginManager.apply(RestApiSpecForTestingPlugin) PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project) configureDependencies(project) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy index 58e72e2dd7be4..8d9517fe42629 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy @@ -18,18 +18,12 @@ */ package org.elasticsearch.gradle.test -import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testclusters.ElasticsearchCluster import org.elasticsearch.gradle.testclusters.RestTestRunnerTask -import org.elasticsearch.gradle.tool.Boilerplate import org.gradle.api.DefaultTask import org.gradle.api.Task -import org.gradle.api.file.FileCopyDetails -import org.gradle.api.tasks.Copy -import org.gradle.api.tasks.Input import org.gradle.api.tasks.testing.Test -import org.gradle.plugins.ide.idea.IdeaPlugin + /** * A wrapper task around setting up a cluster and running rest tests. */ @@ -37,10 +31,6 @@ class RestIntegTestTask extends DefaultTask { protected Test runner - /** Flag indicating whether the rest tests in the rest spec should be run. */ - @Input - Boolean includePackaged = false - RestIntegTestTask() { runner = project.tasks.create("${name}Runner", RestTestRunnerTask.class) super.dependsOn(runner) @@ -69,10 +59,6 @@ class RestIntegTestTask extends DefaultTask { runner.systemProperty('test.clustername', System.getProperty("tests.clustername")) } - // copy the rest spec/tests onto the test classpath - Copy copyRestSpec = createCopyRestSpecTask() - project.sourceSets.test.output.builtBy(copyRestSpec) - // this must run after all projects have been configured, so we know any project // references can be accessed as a fully configured project.gradle.projectsEvaluated { @@ -83,12 +69,6 @@ class RestIntegTestTask extends DefaultTask { } } - /** Sets the includePackaged property */ - public void includePackaged(boolean include) { - includePackaged = include - } - - @Override public Task dependsOn(Object... dependencies) { runner.dependsOn(dependencies) @@ -114,37 +94,4 @@ class RestIntegTestTask extends DefaultTask { project.tasks.getByName("${name}Runner").configure(configure) } - Copy createCopyRestSpecTask() { - Boilerplate.maybeCreate(project.configurations, 'restSpec') { - project.dependencies.add( - 'restSpec', - BuildParams.internal ? project.project(':rest-api-spec') : - "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}" - ) - } - - return Boilerplate.maybeCreate(project.tasks, 'copyRestSpec', Copy) { Copy copy -> - copy.dependsOn project.configurations.restSpec - copy.into(project.sourceSets.test.output.resourcesDir) - copy.from({ project.zipTree(project.configurations.restSpec.singleFile) }) { - includeEmptyDirs = false - include 'rest-api-spec/**' - filesMatching('rest-api-spec/test/**') { FileCopyDetails details -> - if (includePackaged == false) { - details.exclude() - } - } - } - - if (project.plugins.hasPlugin(IdeaPlugin)) { - project.idea { - module { - if (scopes.TEST != null) { - scopes.TEST.plus.add(project.configurations.restSpec) - } - } - } - } - } - } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index b3d323bf3942f..a9f02a4a0448a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -74,6 +74,8 @@ class StandaloneRestTestPlugin implements Plugin { // only setup tests to build SourceSetContainer sourceSets = project.extensions.getByType(SourceSetContainer) SourceSet testSourceSet = sourceSets.create('test') + // need to apply plugin after test source sets are created + project.pluginManager.apply(RestApiSpecForTestingPlugin) project.tasks.withType(Test) { Test test -> test.testClassesDirs = testSourceSet.output.classesDirs diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java index 5b16ca7d40f1e..a662ff4dbcaeb 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java @@ -250,7 +250,7 @@ public void doCheck() throws IOException { Files.write(getSuccessMarker().toPath(), new byte[] {}, StandardOpenOption.CREATE); } else { getLogger().error(problems); - throw new IllegalStateException("Testing conventions are not honored"); + throw new IllegalStateException(String.format("Testing conventions [%s] are not honored", problems)); } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecExtension.java new file mode 100644 index 0000000000000..ff06192c86e14 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecExtension.java @@ -0,0 +1,70 @@ +package org.elasticsearch.gradle.test; + +import java.util.ArrayList; +import java.util.List; + +public class RestApiSpecExtension { + private List includesCoreSpec = new ArrayList<>(); + private List includesCoreTests = new ArrayList<>(); + private List includesXpackSpec = new ArrayList<>(); + private List includesXpackTests = new ArrayList<>(); + private boolean alwaysCopySpec = false; + private boolean copyTests = false; + private boolean copyXpackTests = false; + + public void includeCoreSpec(String include) { + includesCoreSpec.add(include); + } + + public List getIncludesCoreSpec() { + return includesCoreSpec; + } + + public void includeCoreTests(String include) { + includesCoreTests.add(include); + } + + public List getIncludesCoreTests() { + return includesCoreTests; + } + + public void includeXpackSpec(String include) { + includesXpackSpec.add(include); + } + + public List getIncludesXpackSpec() { + return includesXpackSpec; + } + + public void copyCoreTests(boolean copyTests) { + this.copyTests = copyTests; + } + + public boolean shouldCopyCoreTests() { + return copyTests; + } + + public void includeXpackTests(String include) { + includesXpackTests.add(include); + } + + public List getIncludesXpackTests() { + return includesXpackTests; + } + + public void copyXpackTests(boolean copyTests) { + this.copyXpackTests = copyTests; + } + + public boolean shouldCopyXpackTests() { + return copyXpackTests; + } + + public boolean shouldAlwaysCopySpec() { + return alwaysCopySpec; + } + + public void alwaysCopySpec(boolean alwaysCopySpec) { + this.alwaysCopySpec = alwaysCopySpec; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecForTestingPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecForTestingPlugin.java new file mode 100644 index 0000000000000..5b89bb6bb3746 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecForTestingPlugin.java @@ -0,0 +1,141 @@ +package org.elasticsearch.gradle.test; + +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.tool.Boilerplate; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.tasks.Copy; +import org.gradle.api.tasks.SourceSet; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +//TODO: change logger.info to logger.debug +public class RestApiSpecForTestingPlugin implements Plugin { + + private static final Logger logger = Logging.getLogger(RestApiSpecForTestingPlugin.class); + private static final String EXTENSION_NAME = "restApiSpec"; + private static final String apiDir = "rest-api-spec/api"; + private static final String testDir = "rest-api-spec/test"; + + @Override + public void apply(Project project) { + + RestApiSpecExtension extension = project.getExtensions().create(EXTENSION_NAME, RestApiSpecExtension.class); + // need to defer to after evaluation to allow the custom extension to be populated + project.afterEvaluate(p -> { + try { + // copy tests + boolean hasCopiedTests = false; + if (extension.shouldCopyCoreTests()) { + Configuration coreTestConfig = project.getConfigurations().create("copyRestSpecCoreTests"); + Dependency coreTestDep = project.getDependencies() + .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecCoreTests")); + logger.info("Rest specs tests for project [{}] will be copied to the test resources.", project.getPath()); + createCopyTask(project, coreTestConfig, coreTestDep, extension.getIncludesCoreTests(), testDir, false); + hasCopiedTests = true; + } + if (extension.shouldCopyXpackTests()) { + Configuration xpackSpecTestsConfig = project.getConfigurations().create("copyRestSpecXpackTests"); + Dependency xpackTestsDep = project.getDependencies() + .project(Map.of("path", ":x-pack:plugin", "configuration", "restSpecXpackTests")); + logger.info("Rest specs x-pack tests for project [{}] will be copied to the test resources.", project.getPath()); + createCopyTask(project, xpackSpecTestsConfig, xpackTestsDep, extension.getIncludesXpackTests(), testDir, false); + hasCopiedTests = true; + } + // copy specs + if (projectHasRestTests(project) || hasCopiedTests || extension.shouldAlwaysCopySpec()) { + Configuration coreSpecConfig = project.getConfigurations().create("restSpec"); // name chosen for passivity + Dependency coreSpecDep = project.getDependencies() + .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecCore")); + + if (BuildParams.isInternal()) { + // source the specs from this project - this is the path for Elasticsearch builds + if (project.getPath().startsWith(":x-pack")) { + Configuration xpackSpecConfig = project.getConfigurations().create("copyRestSpecXpack"); + Dependency xpackSpecDep = project.getDependencies() + .project(Map.of("path", ":x-pack:plugin", "configuration", "restSpecXpack")); + logger.info("X-pack rest specs for project [{}] will be copied to the test resources.", project.getPath()); + createCopyTask(project, xpackSpecConfig, xpackSpecDep, extension.getIncludesXpackSpec(), apiDir, false); + } + + logger.info("Rest specs for project [{}] will be copied to the test resources.", project.getPath()); + createCopyTask(project, coreSpecConfig, coreSpecDep, extension.getIncludesCoreSpec(), apiDir, false); + + } else { + // source the specs from the published jar - this is the path plugin developers + logger.info( + "Rest specs for project [{}] will be copied to the test resources from the published jar (version: [{}]).", + project.getPath(), + VersionProperties.getElasticsearch() + ); + Dependency coreSpecFromJarDep = project.getDependencies() + .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch()); + + createCopyTask(project, coreSpecConfig, coreSpecFromJarDep, extension.getIncludesCoreSpec(), "", true); + } + } else { + logger.info("Rest specs will be ignored for project [{}] since there are no REST tests", project.getPath()); + return; + } + } catch (Exception e) { + throw new IllegalStateException("Error configuring the rest-api-spec-for-testing plugin. This is likely a bug.", e); + } + }); + } + + private void createCopyTask(Project project, Configuration config, Dependency dep, List includes, String into, boolean isJar) { + project.getDependencies().add(config.getName(), dep); + Copy copyTask = project.getTasks().create(config.getName() + "Task", Copy.class, copy -> { + copy.from(isJar ? project.zipTree(config.getSingleFile()) : config.getSingleFile()); + copy.into(new File(getTestSourceSet(project).getOutput().getResourcesDir(), into)); + if (includes.isEmpty()) { + copy.include(isJar ? apiDir + "/**" : "*/**"); + } else { + includes.forEach(s -> copy.include(isJar ? apiDir + "/" + s + "*/**" : s + "*/**")); + } + copy.setIncludeEmptyDirs(false); + }); + copyTask.dependsOn(config); + project.getTasks().getByName("processTestResources").dependsOn(copyTask); + } + + private boolean projectHasRestTests(Project project) throws IOException { + File testResourceDir = getTestResourceDir(project); + if (testResourceDir == null || new File(testResourceDir, "rest-api-spec/test").exists() == false) { + return false; + } + return Files.walk(testResourceDir.toPath().resolve("rest-api-spec/test")).anyMatch(p -> p.getFileName().toString().endsWith("yml")); + } + + private File getTestResourceDir(Project project) { + SourceSet testSources = getTestSourceSet(project); + if (testSources == null) { + return null; + } + Set resourceDir = testSources.getResources() + .getSrcDirs() + .stream() + .filter(f -> f.isDirectory() && f.getParentFile().getName().equals("test") && f.getName().equals("resources")) + .collect(Collectors.toSet()); + assert resourceDir.size() <= 1; + if (resourceDir.size() == 0) { + return null; + } + return resourceDir.iterator().next(); + } + + private SourceSet getTestSourceSet(Project project) { + return Boilerplate.getJavaSourceSets(project).findByName("test"); + } +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-spec-for-testing.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-spec-for-testing.properties new file mode 100644 index 0000000000000..a38a332550feb --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-spec-for-testing.properties @@ -0,0 +1,20 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +implementation-class=org.elasticsearch.gradle.test.RestApiSpecForTestingPlugin diff --git a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java index a76413d6b47d4..e59bdbe15b61e 100644 --- a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java +++ b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java @@ -147,7 +147,7 @@ private static String propertyName(String methodName) { } public static class MutableBuildParams { - private static MutableBuildParams INSTANCE = new MutableBuildParams(); + public static MutableBuildParams INSTANCE = new MutableBuildParams(); private MutableBuildParams() {} diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index fc15a4550f0a2..5fef82e953fcb 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -24,6 +24,7 @@ apply plugin: 'elasticsearch.rest-test' apply plugin: 'nebula.maven-base-publish' apply plugin: 'nebula.maven-scm' apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'elasticsearch.rest-api-spec-for-testing' group = 'org.elasticsearch.client' archivesBaseName = 'elasticsearch-rest-high-level-client' @@ -36,16 +37,9 @@ publishing { } } -configurations { - restSpec -} - -idea { - module { - if (scopes.TEST != null) { - scopes.TEST.plus.add(project.configurations.restSpec) - } - } +restApiSpec { + //we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions) + alwaysCopySpec true } dependencies { @@ -70,16 +64,9 @@ dependencies { testCompile(project(':x-pack:plugin:core')) { exclude group: 'org.elasticsearch', module: 'elasticsearch-rest-high-level-client' } - - restSpec project(':rest-api-spec') } -//we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions) processTestResources { - dependsOn configurations.restSpec // so that configurations resolve - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } from(project(':client:rest-high-level').file('src/test/resources')) } diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index c72d09adde302..69c0027dc72d4 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -317,9 +317,12 @@ configure(subprojects.findAll { it.name == 'integ-test-zip' }) { group = "org.elasticsearch.distribution.integ-test-zip" + restApiSpec { + copyCoreTests true + } + integTest { dependsOn assemble - includePackaged = true } processTestResources { diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index f9d12ce242da8..246969d6182d6 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -13,13 +13,11 @@ configurations { dockerPlugins dockerSource ossDockerSource - restSpec } dependencies { dockerSource project(path: ":distribution:archives:linux-tar") ossDockerSource project(path: ":distribution:archives:oss-linux-tar") - restSpec project(':rest-api-spec') } ext.expansions = { oss, local -> @@ -128,12 +126,8 @@ preProcessFixture { } processTestResources { - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } from project(':x-pack:plugin:core') .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks') - dependsOn configurations.restSpec } task integTest(type: Test) { diff --git a/docs/build.gradle b/docs/build.gradle index fdb7de3e95d87..bf51a5894eb16 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -41,6 +41,10 @@ buildRestTests.expectedUnconvertedCandidates = [ 'reference/ml/anomaly-detection/apis/update-job.asciidoc' ] +restApiSpec { + alwaysCopySpec true +} + testClusters.integTest { if (singleNode().testDistribution == DEFAULT) { setting 'xpack.license.self_generated.type', 'trial' diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 0f02cbd52d4e9..b122c423707c2 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -30,17 +30,8 @@ tasks.register("bwcTest") { group = 'verification' } -configurations { - restSpec -} - -dependencies { - restSpec project(':rest-api-spec') -} - -processTestResources { - from({ zipTree(configurations.restSpec.singleFile) }) - dependsOn configurations.restSpec +restApiSpec { + copyCoreTests true } for (Version bwcVersion : bwcVersions.wireCompatible) { diff --git a/qa/remote-clusters/build.gradle b/qa/remote-clusters/build.gradle index f3027a0d5b91b..9b8fad8378d5b 100644 --- a/qa/remote-clusters/build.gradle +++ b/qa/remote-clusters/build.gradle @@ -23,12 +23,7 @@ apply plugin: 'elasticsearch.test.fixtures' testFixtures.useFixture() -configurations { - restSpec -} - dependencies { - restSpec project(':rest-api-spec') testCompile project(':client:rest-high-level') } @@ -88,12 +83,8 @@ def createAndSetWritable(Object... locations) { } processTestResources { - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } from project(':x-pack:plugin:core') .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks') - dependsOn configurations.restSpec } task integTest(type: Test) { diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index bf1bc2d5b1073..003fc75910523 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -31,21 +31,6 @@ task bwcTest { group = 'verification' } -configurations { - restSpec -} - -dependencies { - restSpec project(':rest-api-spec') -} - -processTestResources { - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } - dependsOn configurations.restSpec -} - for (Version bwcVersion : bwcVersions.wireCompatible) { /* * The goal here is to: diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle index b76a865c6f898..fff2d77fc614b 100644 --- a/qa/smoke-test-multinode/build.gradle +++ b/qa/smoke-test-multinode/build.gradle @@ -21,8 +21,8 @@ apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' -integTest { - includePackaged = true +restApiSpec { + copyCoreTests true } File repo = file("$buildDir/testclusters/repo") diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index d95ad476682b1..af9385505e9f4 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -4,3 +4,13 @@ apply plugin: 'nebula.maven-scm' test.enabled = false jarHell.enabled = false + +configurations { + restSpecCore + restSpecCoreTests +} + +artifacts { + restSpecCore(new File(projectDir, "src/main/resources/rest-api-spec/api")) + restSpecCoreTests(new File(projectDir, "src/main/resources/rest-api-spec/test")) +} diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index 57759b9427bee..e0dd4a416f357 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -1,5 +1,3 @@ -import java.nio.charset.StandardCharsets - apply plugin: 'elasticsearch.docs-test' /* List of files that have snippets that probably should be converted to @@ -21,12 +19,6 @@ dependencies { testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts') } -// copy xpack rest api -File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') -project.copyRestSpec.from(xpackResources) { - include 'rest-api-spec/api/**' -} - testClusters.integTest { extraConfigFile 'op-jwks.json', xpackProject('test:idp-fixture').file("oidc/op-jwks.json") extraConfigFile 'idp-docs-metadata.xml', xpackProject('test:idp-fixture').file("idp/shibboleth-idp/metadata/idp-docs-metadata.xml") diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 85d28fcd65d40..66e2e9681b10a 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -13,6 +13,13 @@ dependencies { // https://github.com/elastic/x-plugins/issues/724 configurations { testArtifacts.extendsFrom testRuntime + restSpecXpack + restSpecXpackTests +} + +artifacts { + restSpecXpack(new File(projectDir, "src/test/resources/rest-api-spec/api")) + restSpecXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test")) } task testJar(type: Jar) { diff --git a/x-pack/plugin/ccr/qa/build.gradle b/x-pack/plugin/ccr/qa/build.gradle index 46609933699ca..e1c6fb4f95bea 100644 --- a/x-pack/plugin/ccr/qa/build.gradle +++ b/x-pack/plugin/ccr/qa/build.gradle @@ -1,18 +1,6 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - apply plugin: 'elasticsearch.build' test.enabled = false dependencies { compile project(':test:framework') } - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } - -} diff --git a/x-pack/plugin/ccr/qa/rest/build.gradle b/x-pack/plugin/ccr/qa/rest/build.gradle index 4383db34f3e11..d07b265324af0 100644 --- a/x-pack/plugin/ccr/qa/rest/build.gradle +++ b/x-pack/plugin/ccr/qa/rest/build.gradle @@ -3,6 +3,10 @@ import org.elasticsearch.gradle.test.RestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' +restApiSpec { + includeXpackSpec "ccr" +} + dependencies { testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') testCompile project(path: xpackModule('ccr'), configuration: 'runtime') diff --git a/x-pack/plugin/enrich/qa/build.gradle b/x-pack/plugin/enrich/qa/build.gradle index d3e95d997c3fb..79ff4091f6d2d 100644 --- a/x-pack/plugin/enrich/qa/build.gradle +++ b/x-pack/plugin/enrich/qa/build.gradle @@ -6,12 +6,3 @@ test.enabled = false dependencies { compile project(':test:framework') } - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } -} diff --git a/x-pack/plugin/enrich/qa/rest/build.gradle b/x-pack/plugin/enrich/qa/rest/build.gradle index 30ad9ff335e98..2b9a21f3efa94 100644 --- a/x-pack/plugin/enrich/qa/rest/build.gradle +++ b/x-pack/plugin/enrich/qa/rest/build.gradle @@ -2,6 +2,10 @@ apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' +restApiSpec { + includeXpackSpec "enrich" +} + dependencies { testCompile project(path: xpackModule('enrich'), configuration: 'runtime') testCompile project(path: xpackModule('enrich:qa:common'), configuration: 'runtime') diff --git a/x-pack/plugin/eql/qa/build.gradle b/x-pack/plugin/eql/qa/build.gradle index d3e95d997c3fb..79ff4091f6d2d 100644 --- a/x-pack/plugin/eql/qa/build.gradle +++ b/x-pack/plugin/eql/qa/build.gradle @@ -6,12 +6,3 @@ test.enabled = false dependencies { compile project(':test:framework') } - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } -} diff --git a/x-pack/plugin/eql/qa/rest/build.gradle b/x-pack/plugin/eql/qa/rest/build.gradle index 8e4f9dc97e1fa..510d42cba1076 100644 --- a/x-pack/plugin/eql/qa/rest/build.gradle +++ b/x-pack/plugin/eql/qa/rest/build.gradle @@ -4,6 +4,10 @@ apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' +restApiSpec { + includeXpackSpec "eql" +} + dependencies { testCompile project(path: xpackModule('eql'), configuration: 'runtime') testCompile project(path: xpackModule('eql:qa:common'), configuration: 'runtime') diff --git a/x-pack/plugin/graph/qa/build.gradle b/x-pack/plugin/graph/qa/build.gradle index 012e25f5d4f9d..e69de29bb2d1d 100644 --- a/x-pack/plugin/graph/qa/build.gradle +++ b/x-pack/plugin/graph/qa/build.gradle @@ -1,17 +0,0 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - -subprojects { - // HACK: please fix this - // we want to add the rest api specs for xpack to qa tests, but we - // need to wait until after the project is evaluated to only apply - // to those that rest tests. this used to be done automatically - // when xpack was a plugin, but now there is no place with xpack as a module. - // instead, we should package these and make them easy to use for rest tests, - // but currently, they must be copied into the resources of the test runner. - project.tasks.withType(RestIntegTestTask) { - File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xpackResources) { - include 'rest-api-spec/api/**' - } - } -} diff --git a/x-pack/plugin/graph/qa/with-security/build.gradle b/x-pack/plugin/graph/qa/with-security/build.gradle index 578f910146748..fd4cac14529f6 100644 --- a/x-pack/plugin/graph/qa/with-security/build.gradle +++ b/x-pack/plugin/graph/qa/with-security/build.gradle @@ -6,6 +6,10 @@ dependencies { testCompile project(":x-pack:plugin:core") } +restApiSpec { + includeXpackSpec "qa" +} + // bring in graph rest test suite task copyGraphRestTests(type: Copy) { into project.sourceSets.test.output.resourcesDir diff --git a/x-pack/plugin/ilm/qa/build.gradle b/x-pack/plugin/ilm/qa/build.gradle index 46908e1d849b9..e69de29bb2d1d 100644 --- a/x-pack/plugin/ilm/qa/build.gradle +++ b/x-pack/plugin/ilm/qa/build.gradle @@ -1,11 +0,0 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } -} - diff --git a/x-pack/plugin/ilm/qa/rest/build.gradle b/x-pack/plugin/ilm/qa/rest/build.gradle index 3f67169914928..5df1cc17cdb19 100644 --- a/x-pack/plugin/ilm/qa/rest/build.gradle +++ b/x-pack/plugin/ilm/qa/rest/build.gradle @@ -8,6 +8,11 @@ dependencies { testCompile project(path: xpackModule('ilm'), configuration: 'runtime') } +restApiSpec { + includeXpackSpec "ilm" + includeXpackSpec "slm" +} + def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'), password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')] diff --git a/x-pack/plugin/ml/qa/build.gradle b/x-pack/plugin/ml/qa/build.gradle index 012e25f5d4f9d..e69de29bb2d1d 100644 --- a/x-pack/plugin/ml/qa/build.gradle +++ b/x-pack/plugin/ml/qa/build.gradle @@ -1,17 +0,0 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - -subprojects { - // HACK: please fix this - // we want to add the rest api specs for xpack to qa tests, but we - // need to wait until after the project is evaluated to only apply - // to those that rest tests. this used to be done automatically - // when xpack was a plugin, but now there is no place with xpack as a module. - // instead, we should package these and make them easy to use for rest tests, - // but currently, they must be copied into the resources of the test runner. - project.tasks.withType(RestIntegTestTask) { - File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xpackResources) { - include 'rest-api-spec/api/**' - } - } -} diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index fe500ac14abdc..7956de6393a3d 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -9,15 +9,12 @@ dependencies { } // bring in machine learning rest test suite -task copyMlRestTests(type: Copy) { - into project.sourceSets.test.output.resourcesDir - from project(xpackProject('plugin').path).sourceSets.test.resources.srcDirs - include 'rest-api-spec/test/ml/**' +restApiSpec { + copyXpackTests true + includeXpackSpec "ml" + includeXpackTests "ml" } -integTest.runner { - dependsOn copyMlRestTests -} integTest.runner { systemProperty 'tests.rest.blacklist', [ // Remove this test because it doesn't call an ML endpoint and we don't want diff --git a/x-pack/plugin/security/qa/build.gradle b/x-pack/plugin/security/qa/build.gradle index 46908e1d849b9..e69de29bb2d1d 100644 --- a/x-pack/plugin/security/qa/build.gradle +++ b/x-pack/plugin/security/qa/build.gradle @@ -1,11 +0,0 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } -} - diff --git a/x-pack/qa/build.gradle b/x-pack/qa/build.gradle index 2555b0ef729dc..5968a2ce6be71 100644 --- a/x-pack/qa/build.gradle +++ b/x-pack/qa/build.gradle @@ -9,19 +9,3 @@ test.enabled = false dependencies { compile project(':test:framework') } - -subprojects { - // HACK: please fix this - // we want to add the rest api specs for xpack to qa tests, but we - // need to wait until after the project is evaluated to only apply - // to those that rest tests. this used to be done automatically - // when xpack was a plugin, but now there is no place with xpack as a module. - // instead, we should package these and make them easy to use for rest tests, - // but currently, they must be copied into the resources of the test runner. - project.tasks.withType(RestIntegTestTask) { - File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xpackResources) { - include 'rest-api-spec/api/**' - } - } -} diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle index f273efaf5bcd2..30e4f71c106fc 100644 --- a/x-pack/qa/core-rest-tests-with-security/build.gradle +++ b/x-pack/qa/core-rest-tests-with-security/build.gradle @@ -6,8 +6,11 @@ dependencies { testCompile project(':x-pack:qa') } +restApiSpec { + copyCoreTests true +} + integTest { - includePackaged = true runner { systemProperty 'tests.rest.blacklist', [ diff --git a/x-pack/qa/full-cluster-restart/build.gradle b/x-pack/qa/full-cluster-restart/build.gradle index 757d8cd298154..c0527f98766a6 100644 --- a/x-pack/qa/full-cluster-restart/build.gradle +++ b/x-pack/qa/full-cluster-restart/build.gradle @@ -1,8 +1,6 @@ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testclusters.RestTestRunnerTask - apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' @@ -39,24 +37,6 @@ tasks.register("copyTestNodeKeyMaterial", Copy) { into outputDir } -configurations { - restSpec -} - -dependencies { - restSpec project(':rest-api-spec') -} - -processTestResources { - dependsOn configurations.restSpec - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } - from(project(xpackModule('core')).sourceSets.test.resources) { - include 'rest-api-spec/api/**' - } -} - for (Version bwcVersion : bwcVersions.indexCompatible) { String baseName = "v${bwcVersion}" diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle index 9367b74aae379..fd4c78f6ff8b8 100644 --- a/x-pack/qa/rolling-upgrade-basic/build.gradle +++ b/x-pack/qa/rolling-upgrade-basic/build.gradle @@ -1,5 +1,4 @@ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testclusters.RestTestRunnerTask apply plugin: 'elasticsearch.testclusters' @@ -14,25 +13,6 @@ tasks.register("bwcTest") { group = 'verification' } -configurations { - restSpec -} - -dependencies { - restSpec project(':rest-api-spec') -} - -processTestResources { - dependsOn configurations.restSpec - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } - from(project(xpackProject('plugin').path).sourceSets.test.resources) { - include 'rest-api-spec/api/**' - } -} - - for (Version bwcVersion : bwcVersions.wireCompatible) { String baseName = "v${bwcVersion}" diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle index bba1d1fd96e24..5f7c9b6af5662 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle +++ b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle @@ -1,5 +1,4 @@ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testclusters.RestTestRunnerTask apply plugin: 'elasticsearch.testclusters' @@ -14,24 +13,6 @@ tasks.register("bwcTest") { group = 'verification' } -configurations { - restSpec -} - -dependencies { - restSpec project(':rest-api-spec') -} - -processTestResources { - dependsOn configurations.restSpec - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } - from(project(xpackProject('plugin').path).sourceSets.test.resources) { - include 'rest-api-spec/api/**' - } -} - for (Version bwcVersion : bwcVersions.wireCompatible) { String baseName = "v${bwcVersion}" diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 1cd40dd7856b0..9bda4053388d0 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -1,5 +1,4 @@ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testclusters.RestTestRunnerTask apply plugin: 'elasticsearch.testclusters' @@ -23,24 +22,6 @@ tasks.register("bwcTest") { group = 'verification' } -configurations { - restSpec -} - -dependencies { - restSpec project(':rest-api-spec') -} - -processTestResources { - dependsOn configurations.restSpec - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } - from(project(xpackProject('plugin').path).sourceSets.test.resources) { - include 'rest-api-spec/api/**' - } -} - task copyTestNodeKeyMaterial(type: Copy) { from project(':x-pack:plugin:core').files('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem', 'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt') From 1e02fbe8139e4698e1fdf46e29f5602a80ea4207 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Sun, 9 Feb 2020 17:24:15 -0600 Subject: [PATCH 02/23] fix autoscaling project (merge clash) --- x-pack/plugin/autoscaling/qa/build.gradle | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/x-pack/plugin/autoscaling/qa/build.gradle b/x-pack/plugin/autoscaling/qa/build.gradle index 46609933699ca..e1c6fb4f95bea 100644 --- a/x-pack/plugin/autoscaling/qa/build.gradle +++ b/x-pack/plugin/autoscaling/qa/build.gradle @@ -1,18 +1,6 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - apply plugin: 'elasticsearch.build' test.enabled = false dependencies { compile project(':test:framework') } - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } - -} From 02f95b8a814b961fec545e03891d21ac3d509462 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Sun, 9 Feb 2020 17:45:17 -0600 Subject: [PATCH 03/23] fix graph tests --- x-pack/plugin/graph/qa/with-security/build.gradle | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/graph/qa/with-security/build.gradle b/x-pack/plugin/graph/qa/with-security/build.gradle index fd4cac14529f6..786a246fdb597 100644 --- a/x-pack/plugin/graph/qa/with-security/build.gradle +++ b/x-pack/plugin/graph/qa/with-security/build.gradle @@ -6,19 +6,13 @@ dependencies { testCompile project(":x-pack:plugin:core") } -restApiSpec { - includeXpackSpec "qa" -} - // bring in graph rest test suite -task copyGraphRestTests(type: Copy) { - into project.sourceSets.test.output.resourcesDir - from project(xpackProject('plugin').path).sourceSets.test.resources.srcDirs - include 'rest-api-spec/test/graph/**' +restApiSpec { + copyXpackTests true + includeXpackSpec "graph" + includeXpackTests "graph" } -integTest.dependsOn copyGraphRestTests - testClusters.integTest { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'true' From 7b81e65cdc523c14f64a9849a0c13684612943a2 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 10 Feb 2020 08:45:47 -0600 Subject: [PATCH 04/23] fix xpack docs build --- x-pack/docs/build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index e0dd4a416f357..c6039467783d0 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -19,6 +19,10 @@ dependencies { testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts') } +restApiSpec { + alwaysCopySpec true +} + testClusters.integTest { extraConfigFile 'op-jwks.json', xpackProject('test:idp-fixture').file("oidc/op-jwks.json") extraConfigFile 'idp-docs-metadata.xml', xpackProject('test:idp-fixture').file("idp/shibboleth-idp/metadata/idp-docs-metadata.xml") From f334d4050181f0f93aadb7f80a0ed36e07763914 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 10 Feb 2020 09:46:52 -0600 Subject: [PATCH 05/23] fix ml tests --- x-pack/plugin/ml/qa/ml-with-security/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index 7956de6393a3d..e4e4479bfef95 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -12,6 +12,7 @@ dependencies { restApiSpec { copyXpackTests true includeXpackSpec "ml" + includeXpackSpec "cat" includeXpackTests "ml" } From f43429c29594e4bd6e8b0f75d8fbd5d7a96a15ea Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 10 Feb 2020 14:18:55 -0600 Subject: [PATCH 06/23] Use lazy config, task, and review comments --- .../gradle/plugin/PluginBuildPlugin.groovy | 4 +- .../test/StandaloneRestTestPlugin.groovy | 5 +- .../gradle/test/RestApiSpecExtension.java | 70 ------ .../test/RestApiSpecForTestingPlugin.java | 141 ------------ .../test/rest/CopyRestApiExtension.java | 53 +++++ .../gradle/test/rest/CopyRestApiPlugin.java | 93 ++++++++ .../gradle/test/rest/CopyRestApiTask.java | 216 ++++++++++++++++++ ...=> elasticsearch.rest-api-yaml.properties} | 2 +- client/rest-high-level/build.gradle | 7 +- distribution/archives/build.gradle | 4 +- docs/build.gradle | 4 +- qa/mixed-cluster/build.gradle | 4 +- qa/smoke-test-multinode/build.gradle | 4 +- rest-api-spec/build.gradle | 8 +- x-pack/docs/build.gradle | 5 +- x-pack/plugin/build.gradle | 8 +- x-pack/plugin/ccr/qa/rest/build.gradle | 4 +- x-pack/plugin/enrich/qa/rest/build.gradle | 4 +- x-pack/plugin/eql/qa/rest/build.gradle | 4 +- .../graph/qa/with-security/build.gradle | 9 +- x-pack/plugin/ilm/qa/rest/build.gradle | 5 +- .../ml/qa/ml-with-security/build.gradle | 10 +- .../build.gradle | 4 +- 23 files changed, 411 insertions(+), 257 deletions(-) delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecExtension.java delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecForTestingPlugin.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiExtension.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java rename buildSrc/src/main/resources/META-INF/gradle-plugins/{elasticsearch.rest-api-spec-for-testing.properties => elasticsearch.rest-api-yaml.properties} (90%) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index 58407399fdb46..1118ba53ec7ee 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -24,7 +24,7 @@ import org.elasticsearch.gradle.NoticeTask import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.RestApiSpecForTestingPlugin +import org.elasticsearch.gradle.test.rest.CopyRestApiPlugin import org.elasticsearch.gradle.test.RestIntegTestTask import org.elasticsearch.gradle.testclusters.RunTask import org.elasticsearch.gradle.testclusters.TestClustersPlugin @@ -52,7 +52,7 @@ class PluginBuildPlugin implements Plugin { void apply(Project project) { project.pluginManager.apply(BuildPlugin) project.pluginManager.apply(TestClustersPlugin) - project.pluginManager.apply(RestApiSpecForTestingPlugin) + project.pluginManager.apply(CopyRestApiPlugin) PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project) configureDependencies(project) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index a9f02a4a0448a..c75270b8896b7 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -26,9 +26,9 @@ import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin import org.elasticsearch.gradle.precommit.PrecommitTasks +import org.elasticsearch.gradle.test.rest.CopyRestApiPlugin import org.elasticsearch.gradle.testclusters.TestClustersPlugin import org.gradle.api.InvalidUserDataException -import org.gradle.api.JavaVersion import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.artifacts.Configuration @@ -42,6 +42,7 @@ import org.gradle.api.tasks.compile.JavaCompile import org.gradle.api.tasks.testing.Test import org.gradle.plugins.ide.eclipse.model.EclipseModel import org.gradle.plugins.ide.idea.model.IdeaModel + /** * Configures the build to compile tests against Elasticsearch's test framework * and run REST tests. Use BuildPlugin if you want to build main code as well @@ -75,7 +76,7 @@ class StandaloneRestTestPlugin implements Plugin { SourceSetContainer sourceSets = project.extensions.getByType(SourceSetContainer) SourceSet testSourceSet = sourceSets.create('test') // need to apply plugin after test source sets are created - project.pluginManager.apply(RestApiSpecForTestingPlugin) + project.pluginManager.apply(CopyRestApiPlugin) project.tasks.withType(Test) { Test test -> test.testClassesDirs = testSourceSet.output.classesDirs diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecExtension.java deleted file mode 100644 index ff06192c86e14..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecExtension.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.elasticsearch.gradle.test; - -import java.util.ArrayList; -import java.util.List; - -public class RestApiSpecExtension { - private List includesCoreSpec = new ArrayList<>(); - private List includesCoreTests = new ArrayList<>(); - private List includesXpackSpec = new ArrayList<>(); - private List includesXpackTests = new ArrayList<>(); - private boolean alwaysCopySpec = false; - private boolean copyTests = false; - private boolean copyXpackTests = false; - - public void includeCoreSpec(String include) { - includesCoreSpec.add(include); - } - - public List getIncludesCoreSpec() { - return includesCoreSpec; - } - - public void includeCoreTests(String include) { - includesCoreTests.add(include); - } - - public List getIncludesCoreTests() { - return includesCoreTests; - } - - public void includeXpackSpec(String include) { - includesXpackSpec.add(include); - } - - public List getIncludesXpackSpec() { - return includesXpackSpec; - } - - public void copyCoreTests(boolean copyTests) { - this.copyTests = copyTests; - } - - public boolean shouldCopyCoreTests() { - return copyTests; - } - - public void includeXpackTests(String include) { - includesXpackTests.add(include); - } - - public List getIncludesXpackTests() { - return includesXpackTests; - } - - public void copyXpackTests(boolean copyTests) { - this.copyXpackTests = copyTests; - } - - public boolean shouldCopyXpackTests() { - return copyXpackTests; - } - - public boolean shouldAlwaysCopySpec() { - return alwaysCopySpec; - } - - public void alwaysCopySpec(boolean alwaysCopySpec) { - this.alwaysCopySpec = alwaysCopySpec; - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecForTestingPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecForTestingPlugin.java deleted file mode 100644 index 5b89bb6bb3746..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecForTestingPlugin.java +++ /dev/null @@ -1,141 +0,0 @@ -package org.elasticsearch.gradle.test; - -import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.tool.Boilerplate; -import org.gradle.api.Plugin; -import org.gradle.api.Project; -import org.gradle.api.artifacts.Configuration; -import org.gradle.api.artifacts.Dependency; -import org.gradle.api.logging.Logger; -import org.gradle.api.logging.Logging; -import org.gradle.api.tasks.Copy; -import org.gradle.api.tasks.SourceSet; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -//TODO: change logger.info to logger.debug -public class RestApiSpecForTestingPlugin implements Plugin { - - private static final Logger logger = Logging.getLogger(RestApiSpecForTestingPlugin.class); - private static final String EXTENSION_NAME = "restApiSpec"; - private static final String apiDir = "rest-api-spec/api"; - private static final String testDir = "rest-api-spec/test"; - - @Override - public void apply(Project project) { - - RestApiSpecExtension extension = project.getExtensions().create(EXTENSION_NAME, RestApiSpecExtension.class); - // need to defer to after evaluation to allow the custom extension to be populated - project.afterEvaluate(p -> { - try { - // copy tests - boolean hasCopiedTests = false; - if (extension.shouldCopyCoreTests()) { - Configuration coreTestConfig = project.getConfigurations().create("copyRestSpecCoreTests"); - Dependency coreTestDep = project.getDependencies() - .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecCoreTests")); - logger.info("Rest specs tests for project [{}] will be copied to the test resources.", project.getPath()); - createCopyTask(project, coreTestConfig, coreTestDep, extension.getIncludesCoreTests(), testDir, false); - hasCopiedTests = true; - } - if (extension.shouldCopyXpackTests()) { - Configuration xpackSpecTestsConfig = project.getConfigurations().create("copyRestSpecXpackTests"); - Dependency xpackTestsDep = project.getDependencies() - .project(Map.of("path", ":x-pack:plugin", "configuration", "restSpecXpackTests")); - logger.info("Rest specs x-pack tests for project [{}] will be copied to the test resources.", project.getPath()); - createCopyTask(project, xpackSpecTestsConfig, xpackTestsDep, extension.getIncludesXpackTests(), testDir, false); - hasCopiedTests = true; - } - // copy specs - if (projectHasRestTests(project) || hasCopiedTests || extension.shouldAlwaysCopySpec()) { - Configuration coreSpecConfig = project.getConfigurations().create("restSpec"); // name chosen for passivity - Dependency coreSpecDep = project.getDependencies() - .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecCore")); - - if (BuildParams.isInternal()) { - // source the specs from this project - this is the path for Elasticsearch builds - if (project.getPath().startsWith(":x-pack")) { - Configuration xpackSpecConfig = project.getConfigurations().create("copyRestSpecXpack"); - Dependency xpackSpecDep = project.getDependencies() - .project(Map.of("path", ":x-pack:plugin", "configuration", "restSpecXpack")); - logger.info("X-pack rest specs for project [{}] will be copied to the test resources.", project.getPath()); - createCopyTask(project, xpackSpecConfig, xpackSpecDep, extension.getIncludesXpackSpec(), apiDir, false); - } - - logger.info("Rest specs for project [{}] will be copied to the test resources.", project.getPath()); - createCopyTask(project, coreSpecConfig, coreSpecDep, extension.getIncludesCoreSpec(), apiDir, false); - - } else { - // source the specs from the published jar - this is the path plugin developers - logger.info( - "Rest specs for project [{}] will be copied to the test resources from the published jar (version: [{}]).", - project.getPath(), - VersionProperties.getElasticsearch() - ); - Dependency coreSpecFromJarDep = project.getDependencies() - .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch()); - - createCopyTask(project, coreSpecConfig, coreSpecFromJarDep, extension.getIncludesCoreSpec(), "", true); - } - } else { - logger.info("Rest specs will be ignored for project [{}] since there are no REST tests", project.getPath()); - return; - } - } catch (Exception e) { - throw new IllegalStateException("Error configuring the rest-api-spec-for-testing plugin. This is likely a bug.", e); - } - }); - } - - private void createCopyTask(Project project, Configuration config, Dependency dep, List includes, String into, boolean isJar) { - project.getDependencies().add(config.getName(), dep); - Copy copyTask = project.getTasks().create(config.getName() + "Task", Copy.class, copy -> { - copy.from(isJar ? project.zipTree(config.getSingleFile()) : config.getSingleFile()); - copy.into(new File(getTestSourceSet(project).getOutput().getResourcesDir(), into)); - if (includes.isEmpty()) { - copy.include(isJar ? apiDir + "/**" : "*/**"); - } else { - includes.forEach(s -> copy.include(isJar ? apiDir + "/" + s + "*/**" : s + "*/**")); - } - copy.setIncludeEmptyDirs(false); - }); - copyTask.dependsOn(config); - project.getTasks().getByName("processTestResources").dependsOn(copyTask); - } - - private boolean projectHasRestTests(Project project) throws IOException { - File testResourceDir = getTestResourceDir(project); - if (testResourceDir == null || new File(testResourceDir, "rest-api-spec/test").exists() == false) { - return false; - } - return Files.walk(testResourceDir.toPath().resolve("rest-api-spec/test")).anyMatch(p -> p.getFileName().toString().endsWith("yml")); - } - - private File getTestResourceDir(Project project) { - SourceSet testSources = getTestSourceSet(project); - if (testSources == null) { - return null; - } - Set resourceDir = testSources.getResources() - .getSrcDirs() - .stream() - .filter(f -> f.isDirectory() && f.getParentFile().getName().equals("test") && f.getName().equals("resources")) - .collect(Collectors.toSet()); - assert resourceDir.size() <= 1; - if (resourceDir.size() == 0) { - return null; - } - return resourceDir.iterator().next(); - } - - private SourceSet getTestSourceSet(Project project) { - return Boilerplate.getJavaSourceSets(project).findByName("test"); - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiExtension.java new file mode 100644 index 0000000000000..c9af7a58ae833 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiExtension.java @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test.rest; + +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.ListProperty; + +/** + * Custom extension to configure the {@link CopyRestApiTask} + */ +public class CopyRestApiExtension { + + private final ListProperty includeCore; + private final ListProperty includeXpack; + + public CopyRestApiExtension(ObjectFactory objects) { + includeCore = objects.listProperty(String.class); + includeXpack = objects.listProperty(String.class); + } + + public void includeCore(String... include) { + this.includeCore.addAll(include); + } + + public void includeXpack(String... include) { + this.includeXpack.addAll(include); + } + + public ListProperty getIncludeCore() { + return includeCore; + } + + public ListProperty getIncludeXpack() { + return includeXpack; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java new file mode 100644 index 0000000000000..2bc0d7fbd6c9d --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.test.rest; + +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.info.BuildParams; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.provider.Provider; + +import java.util.Map; + +/** + * Gradle plugin to help configure {@link CopyRestApiTask}'s that will copy the artifacts needed for the Rest API spec and YAML tests. + * @see CopyRestApiTask + */ +public class CopyRestApiPlugin implements Plugin { + + private static final String COPY_SPEC_EXTENSION_NAME = "copyRestApiSpecs"; + private static final String COPY_TEST_EXTENSION_NAME = "copyYamlTests"; + + @Override + public void apply(Project project) { + CopyRestApiExtension copySpecExtension = project.getExtensions().create(COPY_SPEC_EXTENSION_NAME, CopyRestApiExtension.class); + CopyRestApiExtension copyTestExtension = project.getExtensions().create(COPY_TEST_EXTENSION_NAME, CopyRestApiExtension.class); + + Provider copyRestYamlTestTask = project.getTasks().register("copyYamlTestsTask", CopyRestApiTask.class, task -> { + task.includeCore.set(copyTestExtension.getIncludeCore()); + task.includeXpack.set(copyTestExtension.getIncludeXpack()); + task.copyTo = "rest-api-spec/test"; + task.coreConfig = project.getConfigurations().create("restSpecTests"); + if (BuildParams.isInternal()) { + Dependency dependency = project.getDependencies().project(Map.of("path", ":rest-api-spec", "configuration", "yamlTests")); + project.getDependencies().add(task.coreConfig.getName(), dependency); + } else { + Dependency dependency = project.getDependencies() + .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch()); + project.getDependencies().add(task.coreConfig.getName(), dependency); + } + task.dependsOn(task.coreConfig); + + task.xpackConfig = project.getConfigurations().create("restSpecTestsXpack"); + Dependency dependency = project.getDependencies().project(Map.of("path", ":x-pack:plugin", "configuration", "yamlXpackTests")); + project.getDependencies().add(task.xpackConfig.getName(), dependency); + task.dependsOn(task.xpackConfig); + }); + + Provider copyRestYamlSpecTask = project.getTasks() + .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> { + task.includeCore.set(copySpecExtension.getIncludeCore()); + task.includeXpack.set(copySpecExtension.getIncludeXpack()); + task.copyTo = "rest-api-spec/api"; + task.dependsOn(copyRestYamlTestTask); + task.coreConfig = project.getConfigurations().create("restSpec"); + if (BuildParams.isInternal()) { + Dependency dependency = project.getDependencies() + .project(Map.of("path", ":rest-api-spec", "configuration", "yamlSpecs")); + project.getDependencies().add(task.coreConfig.getName(), dependency); + } else { + Dependency dependency = project.getDependencies() + .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch()); + project.getDependencies().add(task.coreConfig.getName(), dependency); + } + task.dependsOn(task.coreConfig); + + task.xpackConfig = project.getConfigurations().create("restSpecXpack"); + Dependency dependency = project.getDependencies() + .project(Map.of("path", ":x-pack:plugin", "configuration", "yamlXpackSpecs")); + project.getDependencies().add(task.xpackConfig.getName(), dependency); + task.dependsOn(task.xpackConfig); + }); + + project.getTasks().named("processTestResources").configure(t -> t.dependsOn(copyRestYamlSpecTask)); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java new file mode 100644 index 0000000000000..9852990386f49 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java @@ -0,0 +1,216 @@ +package org.elasticsearch.gradle.test.rest; + +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.tool.Boilerplate; +import org.gradle.api.DefaultTask; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.FileTree; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.SkipWhenEmpty; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.util.PatternFilterable; +import org.gradle.api.tasks.util.PatternSet; +import org.gradle.internal.Factory; + +import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Set; +import java.util.stream.Collectors; + +/** + *

Copies the files needed for the Rest YAML tests to the current projects test resources output directory. + * This is intended to be be used from {@link CopyRestApiPlugin} since the plugin wires up the needed + * configurations and custom extensions. + *

+ *

This task supports copying either the Rest YAML tests (.yml), or the Rest API specification (.json).

+ *
+ * Rest API specification:
+ * When the {@link CopyRestApiPlugin} has been applied this task will automatically copy the Rest API specification + * if there are any Rest YAML tests present (either in source, or output) or if `includeCore` or `includeXpack` has been explicitly + * declared through the 'copyRestApiSpecs' extension.
+ * This task supports copying only a subset of the Rest API specification through the use of the custom extension.
+ * For example: + * 
+ * copyRestApiSpecs {
+ *   includeXpack 'enrich'
+ * }
+ *
+ * Will copy any of the the x-pack specs that start with enrich*. The core API specs will also be copied iff the project also has + * Rest YAML tests. To help optimize the build cache, it is recommended to explicitly declare which specs your project depends on. + * For example: + * 
+ * copyRestApiSpecs {
+ *   includeCore 'index', 'cat'
+ *   includeXpack 'enrich'
+ * }
+ *
+ *
+ * Rest YAML tests :
+ * When the {@link CopyRestApiPlugin} has been applied this task can copy the Rest YAML tests iff explicitly configured with + * `includeCore` or `includeXpack` through the `copyYamlTests` extension. + * For example: + * 
+ * copyYamlTests {
+ *   includeXpack 'graph'
+ * }
+ *
+ * Will copy any of the the x-pack tests that start with graph. + */ +public class CopyRestApiTask extends DefaultTask { + private static final Logger logger = Logging.getLogger(CopyRestApiTask.class); + final ListProperty includeCore = getProject().getObjects().listProperty(String.class); + final ListProperty includeXpack = getProject().getObjects().listProperty(String.class); + + Configuration coreConfig; + Configuration xpackConfig; + String copyTo; + + private final PatternFilterable corePatternSet; + private final PatternFilterable xpackPatternSet; + + public CopyRestApiTask() { + corePatternSet = getPatternSetFactory().create(); + xpackPatternSet = getPatternSetFactory().create(); + } + + @Inject + protected Factory getPatternSetFactory() { + throw new UnsupportedOperationException(); + } + + @Input + public ListProperty getIncludeCore() { + return includeCore; + } + + @Input + public ListProperty getIncludeXpack() { + return includeXpack; + } + + @SkipWhenEmpty + @InputFiles + public FileTree getInputDir() { + xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); + ConfigurableFileCollection fileCollection = getProject().files(xpackConfig.getAsFileTree().matching(xpackPatternSet)); + if (BuildParams.isInternal()) { + corePatternSet.setIncludes(includeCore.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); + fileCollection.plus(coreConfig.getAsFileTree().matching(corePatternSet)); + } else { + fileCollection.plus(coreConfig); + } + return "spec".equals(getTestOrSpec()) && projectHasYamlRestTests() + || includeCore.get().isEmpty() == false + || includeXpack.get().isEmpty() == false ? fileCollection.getAsFileTree() : null; + } + + @OutputDirectory + public File getOutputDir() { + return new File(getTestSourceSet().getOutput().getResourcesDir(), copyTo); + } + + @TaskAction + void copy() { + Project project = getProject(); + if (BuildParams.isInternal()) { + logger.info("Rest {} for project [{}] will be copied to the test resources.", getTestOrSpec(), project.getPath()); + project.copy(c -> { + c.from(coreConfig.getSingleFile()); + c.into(getOutputDir()); + c.include(corePatternSet.getIncludes()); + }); + + } else { + logger.info( + "Rest {} for project [{}] will be copied to the test resources from the published jar (version: [{}]).", + getTestOrSpec(), + project.getPath(), + VersionProperties.getElasticsearch() + ); + project.copy(c -> { + c.from(project.zipTree(coreConfig.getSingleFile())); + c.into(getTestSourceSet().getOutput().getResourcesDir()); // this ends up as the same dir as outputDir + c.include(includeCore.get().stream().map(prefix -> copyTo + "/" + prefix + "*/**").collect(Collectors.toList())); + }); + } + + logger.info("X-pack YAML{} for project [{}] will be copied to the test resources.", getTestOrSpec(), project.getPath()); + project.copy(c -> { + c.from(xpackConfig.getSingleFile()); + c.into(getOutputDir()); + c.include(xpackPatternSet.getIncludes()); + }); + } + + /** + * Returns true if any files with a .yml extension exist the test resources rest-api-spec/test directory (from source or output dir) + */ + private boolean projectHasYamlRestTests() { + File testSourceResourceDir = getTestSourceResourceDir(); + File testOutputResourceDir = getTestOutputResourceDir(); // check output for cases where tests are copied programmatically + + if (testSourceResourceDir == null && testOutputResourceDir == null) { + return false; + } + try { + if (testSourceResourceDir != null) { + return new File(testSourceResourceDir, "rest-api-spec/test").exists() == false + || Files.walk(testSourceResourceDir.toPath().resolve("rest-api-spec/test")) + .anyMatch(p -> p.getFileName().toString().endsWith("yml")); + } + if (testOutputResourceDir != null) { + return new File(testOutputResourceDir, "rest-api-spec/test").exists() == false + || Files.walk(testOutputResourceDir.toPath().resolve("rest-api-spec/test")) + .anyMatch(p -> p.getFileName().toString().endsWith("yml")); + } + } catch (IOException e) { + throw new IllegalStateException(String.format("Error determining if this project [%s] has rest tests.", getProject()), e); + } + return false; + } + + private File getTestSourceResourceDir() { + SourceSet testSources = getTestSourceSet(); + if (testSources == null) { + return null; + } + Set resourceDir = testSources.getResources() + .getSrcDirs() + .stream() + .filter(f -> f.isDirectory() && f.getParentFile().getName().equals("test") && f.getName().equals("resources")) + .collect(Collectors.toSet()); + assert resourceDir.size() <= 1; + if (resourceDir.size() == 0) { + return null; + } + return resourceDir.iterator().next(); + } + + private File getTestOutputResourceDir() { + SourceSet testSources = getTestSourceSet(); + if (testSources == null) { + return null; + } + return testSources.getOutput().getResourcesDir(); + } + + private SourceSet getTestSourceSet() { + return Boilerplate.getJavaSourceSets(getProject()).findByName("test"); + } + + private String getTestOrSpec() { + assert copyTo != null; + return copyTo.endsWith("test") ? "test" : "spec"; + } +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-spec-for-testing.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-yaml.properties similarity index 90% rename from buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-spec-for-testing.properties rename to buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-yaml.properties index a38a332550feb..e2fc92603b519 100644 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-spec-for-testing.properties +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-yaml.properties @@ -17,4 +17,4 @@ # under the License. # -implementation-class=org.elasticsearch.gradle.test.RestApiSpecForTestingPlugin +implementation-class=org.elasticsearch.gradle.test.rest.CopyRestApiPlugin diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index 5fef82e953fcb..fa9b50cc1ae13 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -24,7 +24,7 @@ apply plugin: 'elasticsearch.rest-test' apply plugin: 'nebula.maven-base-publish' apply plugin: 'nebula.maven-scm' apply plugin: 'com.github.johnrengelman.shadow' -apply plugin: 'elasticsearch.rest-api-spec-for-testing' +apply plugin: 'elasticsearch.rest-api-yaml' group = 'org.elasticsearch.client' archivesBaseName = 'elasticsearch-rest-high-level-client' @@ -37,9 +37,10 @@ publishing { } } -restApiSpec { +copyRestApiSpecs { //we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions) - alwaysCopySpec true + includeCore '*' + includeXpack '*' } dependencies { diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 69c0027dc72d4..652c9cd7a7593 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -317,8 +317,8 @@ configure(subprojects.findAll { it.name == 'integ-test-zip' }) { group = "org.elasticsearch.distribution.integ-test-zip" - restApiSpec { - copyCoreTests true + copyYamlTests { + includeCore '*' } integTest { diff --git a/docs/build.gradle b/docs/build.gradle index bf51a5894eb16..cf90e6f1d8167 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -41,8 +41,8 @@ buildRestTests.expectedUnconvertedCandidates = [ 'reference/ml/anomaly-detection/apis/update-job.asciidoc' ] -restApiSpec { - alwaysCopySpec true +copyRestApiSpecs { + includeCore '*' } testClusters.integTest { diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index b122c423707c2..9462ac2df1702 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -30,8 +30,8 @@ tasks.register("bwcTest") { group = 'verification' } -restApiSpec { - copyCoreTests true +copyYamlTests { + includeCore '*' } for (Version bwcVersion : bwcVersions.wireCompatible) { diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle index fff2d77fc614b..2a37346774072 100644 --- a/qa/smoke-test-multinode/build.gradle +++ b/qa/smoke-test-multinode/build.gradle @@ -21,8 +21,8 @@ apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' -restApiSpec { - copyCoreTests true +copyYamlTests { + includeCore '*' } File repo = file("$buildDir/testclusters/repo") diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index af9385505e9f4..ad6c32890f00d 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -6,11 +6,11 @@ test.enabled = false jarHell.enabled = false configurations { - restSpecCore - restSpecCoreTests + yamlSpecs + yamlTests } artifacts { - restSpecCore(new File(projectDir, "src/main/resources/rest-api-spec/api")) - restSpecCoreTests(new File(projectDir, "src/main/resources/rest-api-spec/test")) + yamlSpecs(new File(projectDir, "src/main/resources/rest-api-spec/api")) + yamlTests(new File(projectDir, "src/main/resources/rest-api-spec/test")) } diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index c6039467783d0..f7001864bad37 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -19,8 +19,9 @@ dependencies { testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts') } -restApiSpec { - alwaysCopySpec true +copyRestApiSpecs { + includeCore '*' + includeXpack '*' } testClusters.integTest { diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 66e2e9681b10a..1231147adc349 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -13,13 +13,13 @@ dependencies { // https://github.com/elastic/x-plugins/issues/724 configurations { testArtifacts.extendsFrom testRuntime - restSpecXpack - restSpecXpackTests + yamlXpackSpecs + yamlXpackTests } artifacts { - restSpecXpack(new File(projectDir, "src/test/resources/rest-api-spec/api")) - restSpecXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test")) + yamlXpackSpecs(new File(projectDir, "src/test/resources/rest-api-spec/api")) + yamlXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test")) } task testJar(type: Jar) { diff --git a/x-pack/plugin/ccr/qa/rest/build.gradle b/x-pack/plugin/ccr/qa/rest/build.gradle index d07b265324af0..c1b3be57ef3f8 100644 --- a/x-pack/plugin/ccr/qa/rest/build.gradle +++ b/x-pack/plugin/ccr/qa/rest/build.gradle @@ -3,8 +3,8 @@ import org.elasticsearch.gradle.test.RestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' -restApiSpec { - includeXpackSpec "ccr" +copyRestApiSpecs { + includeXpack 'ccr' } dependencies { diff --git a/x-pack/plugin/enrich/qa/rest/build.gradle b/x-pack/plugin/enrich/qa/rest/build.gradle index 2b9a21f3efa94..edf9975a863ba 100644 --- a/x-pack/plugin/enrich/qa/rest/build.gradle +++ b/x-pack/plugin/enrich/qa/rest/build.gradle @@ -2,8 +2,8 @@ apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' -restApiSpec { - includeXpackSpec "enrich" +copyRestApiSpecs { + includeXpack 'enrich' } dependencies { diff --git a/x-pack/plugin/eql/qa/rest/build.gradle b/x-pack/plugin/eql/qa/rest/build.gradle index 510d42cba1076..36cbad6129d67 100644 --- a/x-pack/plugin/eql/qa/rest/build.gradle +++ b/x-pack/plugin/eql/qa/rest/build.gradle @@ -4,8 +4,8 @@ apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' -restApiSpec { - includeXpackSpec "eql" +copyRestApiSpecs { + includeXpack 'eql' } dependencies { diff --git a/x-pack/plugin/graph/qa/with-security/build.gradle b/x-pack/plugin/graph/qa/with-security/build.gradle index 786a246fdb597..f733a31a668ce 100644 --- a/x-pack/plugin/graph/qa/with-security/build.gradle +++ b/x-pack/plugin/graph/qa/with-security/build.gradle @@ -7,10 +7,11 @@ dependencies { } // bring in graph rest test suite -restApiSpec { - copyXpackTests true - includeXpackSpec "graph" - includeXpackTests "graph" +copyRestApiSpecs { + includeXpack 'graph' +} +copyYamlTests { + includeXpack 'graph' } testClusters.integTest { diff --git a/x-pack/plugin/ilm/qa/rest/build.gradle b/x-pack/plugin/ilm/qa/rest/build.gradle index 5df1cc17cdb19..08f55ba17d25b 100644 --- a/x-pack/plugin/ilm/qa/rest/build.gradle +++ b/x-pack/plugin/ilm/qa/rest/build.gradle @@ -8,9 +8,8 @@ dependencies { testCompile project(path: xpackModule('ilm'), configuration: 'runtime') } -restApiSpec { - includeXpackSpec "ilm" - includeXpackSpec "slm" +copyRestApiSpecs { + includeXpack 'ilm', 'slm' } def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'), diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index e4e4479bfef95..748152e7be6b0 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -9,11 +9,11 @@ dependencies { } // bring in machine learning rest test suite -restApiSpec { - copyXpackTests true - includeXpackSpec "ml" - includeXpackSpec "cat" - includeXpackTests "ml" +copyRestApiSpecs { + includeXpack 'ml', 'cat' +} +copyYamlTests { + includeXpack 'ml' } integTest.runner { diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle index 30e4f71c106fc..dd95cc7a26849 100644 --- a/x-pack/qa/core-rest-tests-with-security/build.gradle +++ b/x-pack/qa/core-rest-tests-with-security/build.gradle @@ -6,8 +6,8 @@ dependencies { testCompile project(':x-pack:qa') } -restApiSpec { - copyCoreTests true +copyYamlTests { + includeCore '*' } integTest { From bddf05bebb3bf720059b7b5a124c009aded4e973 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 12 Feb 2020 13:28:55 -0600 Subject: [PATCH 07/23] change name of plugin --- ...i-yaml.properties => elasticsearch.rest-api-copy.properties} | 0 client/rest-high-level/build.gradle | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename buildSrc/src/main/resources/META-INF/gradle-plugins/{elasticsearch.rest-api-yaml.properties => elasticsearch.rest-api-copy.properties} (100%) diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-yaml.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-copy.properties similarity index 100% rename from buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-yaml.properties rename to buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-copy.properties diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index fa9b50cc1ae13..1792fbcef41a0 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -24,7 +24,7 @@ apply plugin: 'elasticsearch.rest-test' apply plugin: 'nebula.maven-base-publish' apply plugin: 'nebula.maven-scm' apply plugin: 'com.github.johnrengelman.shadow' -apply plugin: 'elasticsearch.rest-api-yaml' +apply plugin: 'elasticsearch.rest-api-copy' group = 'org.elasticsearch.client' archivesBaseName = 'elasticsearch-rest-high-level-client' From 0c1d16403a3c6cb1e743f59e9b4d07fdb3facd58 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 12 Feb 2020 15:21:59 -0600 Subject: [PATCH 08/23] fix hlrc test --- .../gradle/test/rest/CopyRestApiPlugin.java | 14 +++++++------- .../gradle/test/rest/CopyRestApiTask.java | 16 +++++++++------- client/rest-high-level/build.gradle | 1 - rest-api-spec/build.gradle | 8 ++++---- x-pack/plugin/build.gradle | 8 ++++---- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java index 2bc0d7fbd6c9d..75f45646b1d33 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java @@ -46,9 +46,9 @@ public void apply(Project project) { task.includeCore.set(copyTestExtension.getIncludeCore()); task.includeXpack.set(copyTestExtension.getIncludeXpack()); task.copyTo = "rest-api-spec/test"; - task.coreConfig = project.getConfigurations().create("restSpecTests"); + task.coreConfig = project.getConfigurations().create("restTest"); if (BuildParams.isInternal()) { - Dependency dependency = project.getDependencies().project(Map.of("path", ":rest-api-spec", "configuration", "yamlTests")); + Dependency dependency = project.getDependencies().project(Map.of("path", ":rest-api-spec", "configuration", "restTests")); project.getDependencies().add(task.coreConfig.getName(), dependency); } else { Dependency dependency = project.getDependencies() @@ -57,8 +57,8 @@ public void apply(Project project) { } task.dependsOn(task.coreConfig); - task.xpackConfig = project.getConfigurations().create("restSpecTestsXpack"); - Dependency dependency = project.getDependencies().project(Map.of("path", ":x-pack:plugin", "configuration", "yamlXpackTests")); + task.xpackConfig = project.getConfigurations().create("restXpackTest"); + Dependency dependency = project.getDependencies().project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests")); project.getDependencies().add(task.xpackConfig.getName(), dependency); task.dependsOn(task.xpackConfig); }); @@ -72,7 +72,7 @@ public void apply(Project project) { task.coreConfig = project.getConfigurations().create("restSpec"); if (BuildParams.isInternal()) { Dependency dependency = project.getDependencies() - .project(Map.of("path", ":rest-api-spec", "configuration", "yamlSpecs")); + .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs")); project.getDependencies().add(task.coreConfig.getName(), dependency); } else { Dependency dependency = project.getDependencies() @@ -81,9 +81,9 @@ public void apply(Project project) { } task.dependsOn(task.coreConfig); - task.xpackConfig = project.getConfigurations().create("restSpecXpack"); + task.xpackConfig = project.getConfigurations().create("restXpackSpec"); Dependency dependency = project.getDependencies() - .project(Map.of("path", ":x-pack:plugin", "configuration", "yamlXpackSpecs")); + .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs")); project.getDependencies().add(task.xpackConfig.getName(), dependency); task.dependsOn(task.xpackConfig); }); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java index 9852990386f49..93271893b9792 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java @@ -144,13 +144,15 @@ void copy() { c.include(includeCore.get().stream().map(prefix -> copyTo + "/" + prefix + "*/**").collect(Collectors.toList())); }); } - - logger.info("X-pack YAML{} for project [{}] will be copied to the test resources.", getTestOrSpec(), project.getPath()); - project.copy(c -> { - c.from(xpackConfig.getSingleFile()); - c.into(getOutputDir()); - c.include(xpackPatternSet.getIncludes()); - }); + // only include x-pack if explicitly instructed + if (includeXpack.get().isEmpty() == false) { + logger.info("X-pack YAML{} for project [{}] will be copied to the test resources.", getTestOrSpec(), project.getPath()); + project.copy(c -> { + c.from(xpackConfig.getSingleFile()); + c.into(getOutputDir()); + c.include(xpackPatternSet.getIncludes()); + }); + } } /** diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index 1792fbcef41a0..6ec34ea95662e 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -40,7 +40,6 @@ publishing { copyRestApiSpecs { //we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions) includeCore '*' - includeXpack '*' } dependencies { diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index ad6c32890f00d..fa29345e0ff6b 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -6,11 +6,11 @@ test.enabled = false jarHell.enabled = false configurations { - yamlSpecs - yamlTests + restSpecs + restTests } artifacts { - yamlSpecs(new File(projectDir, "src/main/resources/rest-api-spec/api")) - yamlTests(new File(projectDir, "src/main/resources/rest-api-spec/test")) + restSpecs(new File(projectDir, "src/main/resources/rest-api-spec/api")) + restTests(new File(projectDir, "src/main/resources/rest-api-spec/test")) } diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 1231147adc349..f2ca68bd81416 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -13,13 +13,13 @@ dependencies { // https://github.com/elastic/x-plugins/issues/724 configurations { testArtifacts.extendsFrom testRuntime - yamlXpackSpecs - yamlXpackTests + restXpackSpecs + restXpackTests } artifacts { - yamlXpackSpecs(new File(projectDir, "src/test/resources/rest-api-spec/api")) - yamlXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test")) + restXpackSpecs(new File(projectDir, "src/test/resources/rest-api-spec/api")) + restXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test")) } task testJar(type: Jar) { From bbcde3abfce4b3852af9be693f2724e1d0a76c5b Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 12 Feb 2020 16:33:52 -0600 Subject: [PATCH 09/23] fix some missed includes --- x-pack/plugin/autoscaling/qa/build.gradle | 4 ++++ x-pack/qa/rolling-upgrade/build.gradle | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/x-pack/plugin/autoscaling/qa/build.gradle b/x-pack/plugin/autoscaling/qa/build.gradle index e1c6fb4f95bea..e0f6727d10894 100644 --- a/x-pack/plugin/autoscaling/qa/build.gradle +++ b/x-pack/plugin/autoscaling/qa/build.gradle @@ -4,3 +4,7 @@ test.enabled = false dependencies { compile project(':test:framework') } + +copyRestApiSpecs { + includeXpack 'autoscaling' +} diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 9bda4053388d0..430e4d6e3c3b2 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -9,6 +9,11 @@ dependencies { testCompile project(':client:rest-high-level') } +copyRestApiSpecs { + includeCore '*' + includeXpack '*' +} + compileTestJava.options.compilerArgs << "-Xlint:-cast,-rawtypes,-unchecked" forbiddenPatterns { From 2dc1a3af361cb8a41fcec193ea81e58a47989e5a Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 12 Feb 2020 17:29:48 -0600 Subject: [PATCH 10/23] move to 1 extension, minor clean up --- .../gradle/test/rest/CopyRestApiPlugin.java | 15 +++--- .../gradle/test/rest/CopyRestApiTask.java | 19 ++++++++ .../test/rest/RestResourcesExtension.java | 47 +++++++++++++++++++ ...iExtension.java => RestResourcesSpec.java} | 8 +--- client/rest-high-level/build.gradle | 6 ++- distribution/archives/build.gradle | 6 ++- docs/build.gradle | 6 ++- qa/mixed-cluster/build.gradle | 6 ++- qa/smoke-test-multinode/build.gradle | 6 ++- x-pack/docs/build.gradle | 8 ++-- x-pack/plugin/autoscaling/qa/build.gradle | 4 -- .../plugin/autoscaling/qa/rest/build.gradle | 6 +++ x-pack/plugin/ccr/qa/rest/build.gradle | 6 ++- x-pack/plugin/enrich/qa/rest/build.gradle | 6 ++- x-pack/plugin/eql/qa/rest/build.gradle | 6 ++- .../graph/qa/with-security/build.gradle | 12 +++-- x-pack/plugin/ilm/qa/rest/build.gradle | 6 ++- .../ml/qa/ml-with-security/build.gradle | 12 +++-- .../build.gradle | 6 ++- x-pack/qa/rolling-upgrade/build.gradle | 8 ++-- 20 files changed, 144 insertions(+), 55 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java rename buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/{CopyRestApiExtension.java => RestResourcesSpec.java} (90%) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java index 75f45646b1d33..3a8be47afea12 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.elasticsearch.gradle.test.rest; import org.elasticsearch.gradle.VersionProperties; @@ -34,17 +33,15 @@ */ public class CopyRestApiPlugin implements Plugin { - private static final String COPY_SPEC_EXTENSION_NAME = "copyRestApiSpecs"; - private static final String COPY_TEST_EXTENSION_NAME = "copyYamlTests"; + private static final String EXTENSION_NAME = "restResources"; @Override public void apply(Project project) { - CopyRestApiExtension copySpecExtension = project.getExtensions().create(COPY_SPEC_EXTENSION_NAME, CopyRestApiExtension.class); - CopyRestApiExtension copyTestExtension = project.getExtensions().create(COPY_TEST_EXTENSION_NAME, CopyRestApiExtension.class); + RestResourcesExtension extension = project.getExtensions().create(EXTENSION_NAME, RestResourcesExtension.class); Provider copyRestYamlTestTask = project.getTasks().register("copyYamlTestsTask", CopyRestApiTask.class, task -> { - task.includeCore.set(copyTestExtension.getIncludeCore()); - task.includeXpack.set(copyTestExtension.getIncludeXpack()); + task.includeCore.set(extension.restTests.getIncludeCore()); + task.includeXpack.set(extension.restTests.getIncludeXpack()); task.copyTo = "rest-api-spec/test"; task.coreConfig = project.getConfigurations().create("restTest"); if (BuildParams.isInternal()) { @@ -65,8 +62,8 @@ public void apply(Project project) { Provider copyRestYamlSpecTask = project.getTasks() .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> { - task.includeCore.set(copySpecExtension.getIncludeCore()); - task.includeXpack.set(copySpecExtension.getIncludeXpack()); + task.includeCore.set(extension.restApi.getIncludeCore()); + task.includeXpack.set(extension.restApi.getIncludeXpack()); task.copyTo = "rest-api-spec/api"; task.dependsOn(copyRestYamlTestTask); task.coreConfig = project.getConfigurations().create("restSpec"); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java index 93271893b9792..c094b02f9cd5a 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java @@ -1,3 +1,21 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.elasticsearch.gradle.test.rest; import org.elasticsearch.gradle.VersionProperties; @@ -40,6 +58,7 @@ * if there are any Rest YAML tests present (either in source, or output) or if `includeCore` or `includeXpack` has been explicitly * declared through the 'copyRestApiSpecs' extension.
* This task supports copying only a subset of the Rest API specification through the use of the custom extension.
+ * //TODO: fix the examples here! * For example: * 
  * copyRestApiSpecs {
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java
new file mode 100644
index 0000000000000..df51af42fe13a
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.test.rest;
+
+import org.gradle.api.Action;
+import org.gradle.api.model.ObjectFactory;
+
+import javax.inject.Inject;
+
+/**
+ * Custom extension to configure the {@link CopyRestApiTask}
+ */
+public class RestResourcesExtension {
+
+    final RestResourcesSpec restApi;
+    final RestResourcesSpec restTests;
+
+    @Inject
+    public RestResourcesExtension(ObjectFactory objects) {
+        restApi = new RestResourcesSpec(objects);
+        restTests = new RestResourcesSpec(objects);
+    }
+
+    void restApi(Action spec) {
+        spec.execute(restApi);
+    }
+
+    void restTests(Action spec) {
+        spec.execute(restTests);
+    }
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesSpec.java
similarity index 90%
rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiExtension.java
rename to buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesSpec.java
index c9af7a58ae833..49e7b62731a8b 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiExtension.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesSpec.java
@@ -16,21 +16,17 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
 package org.elasticsearch.gradle.test.rest;
 
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.ListProperty;
 
-/**
- * Custom extension to configure the {@link CopyRestApiTask}
- */
-public class CopyRestApiExtension {
+public class RestResourcesSpec {
 
     private final ListProperty includeCore;
     private final ListProperty includeXpack;
 
-    public CopyRestApiExtension(ObjectFactory objects) {
+    public RestResourcesSpec(ObjectFactory objects) {
         includeCore = objects.listProperty(String.class);
         includeXpack = objects.listProperty(String.class);
     }
diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
index 6ec34ea95662e..be7f29500e0d2 100644
--- a/client/rest-high-level/build.gradle
+++ b/client/rest-high-level/build.gradle
@@ -37,9 +37,11 @@ publishing {
   }
 }
 
-copyRestApiSpecs {
+restResources {
   //we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions)
-  includeCore '*'
+  restApi {
+    includeCore '*'
+  }
 }
 
 dependencies {
diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle
index 652c9cd7a7593..029eb0733023f 100644
--- a/distribution/archives/build.gradle
+++ b/distribution/archives/build.gradle
@@ -317,8 +317,10 @@ configure(subprojects.findAll { it.name == 'integ-test-zip' }) {
 
   group = "org.elasticsearch.distribution.integ-test-zip"
 
-  copyYamlTests {
-    includeCore '*'
+  restResources {
+    restTests {
+      includeCore '*'
+    }
   }
 
   integTest {
diff --git a/docs/build.gradle b/docs/build.gradle
index 5f77d5faa2d90..0deeb0ea9cdd2 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -41,8 +41,10 @@ buildRestTests.expectedUnconvertedCandidates = [
   'reference/ml/anomaly-detection/apis/update-job.asciidoc'
 ]
 
-copyRestApiSpecs {
-  includeCore '*'
+restResources {
+  restApi {
+    includeCore '*'
+  }
 }
 
 testClusters.integTest {
diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle
index 9462ac2df1702..ef9d05bf65744 100644
--- a/qa/mixed-cluster/build.gradle
+++ b/qa/mixed-cluster/build.gradle
@@ -30,8 +30,10 @@ tasks.register("bwcTest") {
   group = 'verification'
 }
 
-copyYamlTests {
-  includeCore '*'
+restResources {
+  restTests {
+    includeCore '*'
+  }
 }
 
 for (Version bwcVersion : bwcVersions.wireCompatible) {
diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle
index 2a37346774072..244b5d1e8af6e 100644
--- a/qa/smoke-test-multinode/build.gradle
+++ b/qa/smoke-test-multinode/build.gradle
@@ -21,8 +21,10 @@ apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-copyYamlTests {
-  includeCore '*'
+restResources {
+  restTests {
+    includeCore '*'
+  }
 }
 
 File repo = file("$buildDir/testclusters/repo")
diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle
index f7001864bad37..c7a82ba62ec3e 100644
--- a/x-pack/docs/build.gradle
+++ b/x-pack/docs/build.gradle
@@ -19,9 +19,11 @@ dependencies {
   testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
 }
 
-copyRestApiSpecs {
-  includeCore '*'
-  includeXpack '*'
+restResources {
+  restApi {
+    includeCore '*'
+    includeXpack '*'
+  }
 }
 
 testClusters.integTest {
diff --git a/x-pack/plugin/autoscaling/qa/build.gradle b/x-pack/plugin/autoscaling/qa/build.gradle
index e0f6727d10894..e1c6fb4f95bea 100644
--- a/x-pack/plugin/autoscaling/qa/build.gradle
+++ b/x-pack/plugin/autoscaling/qa/build.gradle
@@ -4,7 +4,3 @@ test.enabled = false
 dependencies {
   compile project(':test:framework')
 }
-
-copyRestApiSpecs {
-  includeXpack 'autoscaling'
-}
diff --git a/x-pack/plugin/autoscaling/qa/rest/build.gradle b/x-pack/plugin/autoscaling/qa/rest/build.gradle
index 9e297a87ea18d..631c6e950ee1a 100644
--- a/x-pack/plugin/autoscaling/qa/rest/build.gradle
+++ b/x-pack/plugin/autoscaling/qa/rest/build.gradle
@@ -9,6 +9,12 @@ dependencies {
   testCompile project(path: xpackModule('autoscaling'), configuration: 'runtime')
 }
 
+restResources {
+  restApi {
+    includeXpack 'autoscaling'
+  }
+}
+
 task restTest(type: RestIntegTestTask) {
   mustRunAfter(precommit)
 }
diff --git a/x-pack/plugin/ccr/qa/rest/build.gradle b/x-pack/plugin/ccr/qa/rest/build.gradle
index c1b3be57ef3f8..d95ac271df519 100644
--- a/x-pack/plugin/ccr/qa/rest/build.gradle
+++ b/x-pack/plugin/ccr/qa/rest/build.gradle
@@ -3,8 +3,10 @@ import org.elasticsearch.gradle.test.RestIntegTestTask
 apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
-copyRestApiSpecs {
-  includeXpack 'ccr'
+restResources {
+  restApi {
+    includeXpack 'ccr'
+  }
 }
 
 dependencies {
diff --git a/x-pack/plugin/enrich/qa/rest/build.gradle b/x-pack/plugin/enrich/qa/rest/build.gradle
index edf9975a863ba..209154dd448b0 100644
--- a/x-pack/plugin/enrich/qa/rest/build.gradle
+++ b/x-pack/plugin/enrich/qa/rest/build.gradle
@@ -2,8 +2,10 @@ apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-copyRestApiSpecs {
-  includeXpack 'enrich'
+restResources {
+  restApi {
+    includeXpack 'enrich'
+  }
 }
 
 dependencies {
diff --git a/x-pack/plugin/eql/qa/rest/build.gradle b/x-pack/plugin/eql/qa/rest/build.gradle
index 36cbad6129d67..a7e94fcba6d6d 100644
--- a/x-pack/plugin/eql/qa/rest/build.gradle
+++ b/x-pack/plugin/eql/qa/rest/build.gradle
@@ -4,8 +4,10 @@ apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-copyRestApiSpecs {
-  includeXpack 'eql'
+restResources {
+  restApi {
+    includeXpack 'eql'
+  }
 }
 
 dependencies {
diff --git a/x-pack/plugin/graph/qa/with-security/build.gradle b/x-pack/plugin/graph/qa/with-security/build.gradle
index f733a31a668ce..869e926102dcb 100644
--- a/x-pack/plugin/graph/qa/with-security/build.gradle
+++ b/x-pack/plugin/graph/qa/with-security/build.gradle
@@ -7,11 +7,13 @@ dependencies {
 }
 
 // bring in graph rest test suite
-copyRestApiSpecs {
-  includeXpack 'graph'
-}
-copyYamlTests {
-  includeXpack 'graph'
+restResources {
+  restApi {
+    includeXpack 'graph'
+  }
+  restTests {
+    includeXpack 'graph'
+  }
 }
 
 testClusters.integTest {
diff --git a/x-pack/plugin/ilm/qa/rest/build.gradle b/x-pack/plugin/ilm/qa/rest/build.gradle
index 08f55ba17d25b..6c5ea98a0b46d 100644
--- a/x-pack/plugin/ilm/qa/rest/build.gradle
+++ b/x-pack/plugin/ilm/qa/rest/build.gradle
@@ -8,8 +8,10 @@ dependencies {
   testCompile project(path: xpackModule('ilm'), configuration: 'runtime')
 }
 
-copyRestApiSpecs {
-  includeXpack 'ilm', 'slm'
+restResources {
+  restApi {
+    includeXpack 'ilm', 'slm'
+  }
 }
 
 def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'),
diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
index 748152e7be6b0..5fbc7d87cce48 100644
--- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle
+++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
@@ -9,11 +9,13 @@ dependencies {
 }
 
 // bring in machine learning rest test suite
-copyRestApiSpecs {
-  includeXpack 'ml', 'cat'
-}
-copyYamlTests {
-  includeXpack 'ml'
+restResources {
+  restApi {
+    includeXpack 'ml', 'cat'
+  }
+  restTests {
+    includeXpack 'ml'
+  }
 }
 
 integTest.runner {
diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle
index dd95cc7a26849..4fc4428c8ed6f 100644
--- a/x-pack/qa/core-rest-tests-with-security/build.gradle
+++ b/x-pack/qa/core-rest-tests-with-security/build.gradle
@@ -6,8 +6,10 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
-copyYamlTests {
-  includeCore '*'
+restResources {
+  restTests {
+    includeCore '*'
+  }
 }
 
 integTest {
diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle
index 430e4d6e3c3b2..cc63a743cf9ae 100644
--- a/x-pack/qa/rolling-upgrade/build.gradle
+++ b/x-pack/qa/rolling-upgrade/build.gradle
@@ -9,9 +9,11 @@ dependencies {
   testCompile project(':client:rest-high-level')
 }
 
-copyRestApiSpecs {
-  includeCore '*'
-  includeXpack '*'
+restResources {
+  restApi {
+    includeCore '*'
+    includeXpack '*'
+  }
 }
 
 compileTestJava.options.compilerArgs << "-Xlint:-cast,-rawtypes,-unchecked"

From 8942ac214e8f886f9943066b21b6676fe2b8102a Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Thu, 13 Feb 2020 08:59:51 -0600
Subject: [PATCH 11/23] split to two tasks

---
 .../gradle/test/rest/CopyRestApiPlugin.java   | 100 +++++++---
 .../gradle/test/rest/CopyRestApiTask.java     |  63 +-----
 .../gradle/test/rest/CopyRestTestsTask.java   | 185 ++++++++++++++++++
 3 files changed, 270 insertions(+), 78 deletions(-)
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java
index 3a8be47afea12..721ac4901911b 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java
@@ -27,9 +27,57 @@
 
 import java.util.Map;
 
+//TODO: clean up this doc!
 /**
+ * 

  * Gradle plugin to help configure {@link CopyRestApiTask}'s that will copy the artifacts needed for the Rest API spec and YAML tests.
+ * 

+ * 
Copies the files needed for the Rest YAML tests to the current projects test resources output directory.
+ * This is intended to be be used from {@link CopyRestApiPlugin} since the plugin wires up the needed
+ * configurations and custom extensions.
+ * 

+ * 
This task supports copying either the Rest YAML tests (.yml), or the Rest API specification (.json).

+ * 

+ * Rest API specification: 

+ * When the {@link CopyRestApiPlugin} has been applied this task will automatically copy the Rest API specification
+ * if there are any Rest YAML tests present (either in source, or output) or if `restApi.includeCore` or `restApi.includeXpack` has been
+ * explicitly declared through the 'restResources' extension. 

+ * This task supports copying only a subset of the Rest API specification through the use of the custom extension.

+ * For example:
+ * 
+ * restResources {
+ *   restApi {
+ *     includeXpack 'enrich'
+ *   }
+ * }
+ * 

+ * Will copy any of the the x-pack specs that start with enrich*. The core API specs will also be copied iff the project also has
+ * Rest YAML tests. To help optimize the build cache, it is recommended to explicitly declare which specs your project depends on.
+ * For example:
+ * 
+ * restResources {
+ *   restApi {
+ *     includeCore 'index', 'cat'
+ *     includeXpack 'enrich'
+ *   }
+ * }
+ * 

+ * 

+ * Rest YAML tests : 

+ * When the {@link CopyRestApiPlugin} has been applied this task can copy the Rest YAML tests iff explicitly configured with
+ * `includeCore` or `includeXpack` through the `restResources.restTests` extension.
+ * For example:
+ * 
+ * restResources {
+ *   restTests {
+ *     includeXpack 'graph'
+ *   }
+ * }
+ * 

+ * Will copy any of the the x-pack tests that start with graph.
+ *
  * @see CopyRestApiTask
+ * @see CopyRestTestsTask
  */
 public class CopyRestApiPlugin implements Plugin {
 
@@ -39,26 +87,29 @@ public class CopyRestApiPlugin implements Plugin {
     public void apply(Project project) {
         RestResourcesExtension extension = project.getExtensions().create(EXTENSION_NAME, RestResourcesExtension.class);
 
-        Provider copyRestYamlTestTask = project.getTasks().register("copyYamlTestsTask", CopyRestApiTask.class, task -> {
-            task.includeCore.set(extension.restTests.getIncludeCore());
-            task.includeXpack.set(extension.restTests.getIncludeXpack());
-            task.copyTo = "rest-api-spec/test";
-            task.coreConfig = project.getConfigurations().create("restTest");
-            if (BuildParams.isInternal()) {
-                Dependency dependency = project.getDependencies().project(Map.of("path", ":rest-api-spec", "configuration", "restTests"));
-                project.getDependencies().add(task.coreConfig.getName(), dependency);
-            } else {
-                Dependency dependency = project.getDependencies()
-                    .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
-                project.getDependencies().add(task.coreConfig.getName(), dependency);
-            }
-            task.dependsOn(task.coreConfig);
+        Provider copyRestYamlTestTask = project.getTasks()
+            .register("copyYamlTestsTask", CopyRestTestsTask.class, task -> {
+                task.includeCore.set(extension.restTests.getIncludeCore());
+                task.includeXpack.set(extension.restTests.getIncludeXpack());
+                task.copyTo = "rest-api-spec/test";
+                task.coreConfig = project.getConfigurations().create("restTest");
+                if (BuildParams.isInternal()) {
+                    Dependency dependency = project.getDependencies()
+                        .project(Map.of("path", ":rest-api-spec", "configuration", "restTests"));
+                    project.getDependencies().add(task.coreConfig.getName(), dependency);
 
-            task.xpackConfig = project.getConfigurations().create("restXpackTest");
-            Dependency dependency = project.getDependencies().project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
-            project.getDependencies().add(task.xpackConfig.getName(), dependency);
-            task.dependsOn(task.xpackConfig);
-        });
+                    task.xpackConfig = project.getConfigurations().create("restXpackTest");
+                    dependency = project.getDependencies().project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
+                    project.getDependencies().add(task.xpackConfig.getName(), dependency);
+                    task.dependsOn(task.xpackConfig);
+
+                } else {
+                    Dependency dependency = project.getDependencies()
+                        .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
+                    project.getDependencies().add(task.coreConfig.getName(), dependency);
+                }
+                task.dependsOn(task.coreConfig);
+            });
 
         Provider copyRestYamlSpecTask = project.getTasks()
             .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> {
@@ -71,18 +122,17 @@ public void apply(Project project) {
                     Dependency dependency = project.getDependencies()
                         .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs"));
                     project.getDependencies().add(task.coreConfig.getName(), dependency);
+
+                    task.xpackConfig = project.getConfigurations().create("restXpackSpec");
+                    dependency = project.getDependencies().project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs"));
+                    project.getDependencies().add(task.xpackConfig.getName(), dependency);
+                    task.dependsOn(task.xpackConfig);
                 } else {
                     Dependency dependency = project.getDependencies()
                         .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
                     project.getDependencies().add(task.coreConfig.getName(), dependency);
                 }
                 task.dependsOn(task.coreConfig);
-
-                task.xpackConfig = project.getConfigurations().create("restXpackSpec");
-                Dependency dependency = project.getDependencies()
-                    .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs"));
-                project.getDependencies().add(task.xpackConfig.getName(), dependency);
-                task.dependsOn(task.xpackConfig);
             });
 
         project.getTasks().named("processTestResources").configure(t -> t.dependsOn(copyRestYamlSpecTask));
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
index c094b02f9cd5a..c40b95811cbf4 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
@@ -46,46 +46,6 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-/**
- * 
Copies the files needed for the Rest YAML tests to the current projects test resources output directory.
- * This is intended to be be used from {@link CopyRestApiPlugin} since the plugin wires up the needed
- * configurations and custom extensions.
- * 

- * 
This task supports copying either the Rest YAML tests (.yml), or the Rest API specification (.json).

- * 

- * Rest API specification: 

- * When the {@link CopyRestApiPlugin} has been applied this task will automatically copy the Rest API specification
- * if there are any Rest YAML tests present (either in source, or output) or if `includeCore` or `includeXpack` has been explicitly
- * declared through the 'copyRestApiSpecs' extension. 

- * This task supports copying only a subset of the Rest API specification through the use of the custom extension.

- *     //TODO: fix the examples here!
- * For example:
- * 
- * copyRestApiSpecs {
- *   includeXpack 'enrich'
- * }
- * 

- * Will copy any of the the x-pack specs that start with enrich*. The core API specs will also be copied iff the project also has
- * Rest YAML tests. To help optimize the build cache, it is recommended to explicitly declare which specs your project depends on.
- * For example:
- * 
- * copyRestApiSpecs {
- *   includeCore 'index', 'cat'
- *   includeXpack 'enrich'
- * }
- *  

- * 

- * Rest YAML tests : 

- * When the {@link CopyRestApiPlugin} has been applied this task can copy the Rest YAML tests iff explicitly configured with
- * `includeCore` or `includeXpack` through the `copyYamlTests` extension.
- * For example:
- * 
- * copyYamlTests {
- *   includeXpack 'graph'
- * }
- * 

- * Will copy any of the the x-pack tests that start with graph.
- */
 public class CopyRestApiTask extends DefaultTask {
     private static final Logger logger = Logging.getLogger(CopyRestApiTask.class);
     final ListProperty includeCore = getProject().getObjects().listProperty(String.class);
@@ -99,6 +59,7 @@ public class CopyRestApiTask extends DefaultTask {
     private final PatternFilterable xpackPatternSet;
 
     public CopyRestApiTask() {
+        // TODO: blow up if internal and requested x-pack
         corePatternSet = getPatternSetFactory().create();
         xpackPatternSet = getPatternSetFactory().create();
     }
@@ -129,9 +90,10 @@ public FileTree getInputDir() {
         } else {
             fileCollection.plus(coreConfig);
         }
-        return "spec".equals(getTestOrSpec()) && projectHasYamlRestTests()
-            || includeCore.get().isEmpty() == false
-            || includeXpack.get().isEmpty() == false ? fileCollection.getAsFileTree() : null;
+        // if project has rest tests or the includes are explicitly configured execute the task, else NO-SOURCE due to the null input
+        return projectHasYamlRestTests() || includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false
+            ? fileCollection.getAsFileTree()
+            : null;
     }
 
     @OutputDirectory
@@ -142,8 +104,9 @@ public File getOutputDir() {
     @TaskAction
     void copy() {
         Project project = getProject();
+        // always copy the core specs if the task executes
         if (BuildParams.isInternal()) {
-            logger.info("Rest {} for project [{}] will be copied to the test resources.", getTestOrSpec(), project.getPath());
+            logger.info("Rest specs for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
                 c.from(coreConfig.getSingleFile());
                 c.into(getOutputDir());
@@ -152,8 +115,7 @@ void copy() {
 
         } else {
             logger.info(
-                "Rest {} for project [{}] will be copied to the test resources from the published jar (version: [{}]).",
-                getTestOrSpec(),
+                "Rest specs for project [{}] will be copied to the test resources from the published jar (version: [{}]).",
                 project.getPath(),
                 VersionProperties.getElasticsearch()
             );
@@ -163,9 +125,9 @@ void copy() {
                 c.include(includeCore.get().stream().map(prefix -> copyTo + "/" + prefix + "*/**").collect(Collectors.toList()));
             });
         }
-        // only include x-pack if explicitly instructed
+        // only copy x-pack specs if explicitly instructed
         if (includeXpack.get().isEmpty() == false) {
-            logger.info("X-pack YAML{} for project [{}] will be copied to the test resources.", getTestOrSpec(), project.getPath());
+            logger.info("X-pack rest specs for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
                 c.from(xpackConfig.getSingleFile());
                 c.into(getOutputDir());
@@ -229,9 +191,4 @@ private File getTestOutputResourceDir() {
     private SourceSet getTestSourceSet() {
         return Boilerplate.getJavaSourceSets(getProject()).findByName("test");
     }
-
-    private String getTestOrSpec() {
-        assert copyTo != null;
-        return copyTo.endsWith("test") ? "test" : "spec";
-    }
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
new file mode 100644
index 0000000000000..c7202f3377742
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.test.rest;
+
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.info.BuildParams;
+import org.elasticsearch.gradle.tool.Boilerplate;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.FileTree;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.provider.ListProperty;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.SkipWhenEmpty;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.api.tasks.util.PatternFilterable;
+import org.gradle.api.tasks.util.PatternSet;
+import org.gradle.internal.Factory;
+
+import javax.inject.Inject;
+import java.io.File;
+import java.util.stream.Collectors;
+
+/**
+ * 
Copies the files needed for the Rest YAML tests to the current projects test resources output directory.
+ * This is intended to be be used from {@link CopyRestApiPlugin} since the plugin wires up the needed
+ * configurations and custom extensions.
+ * 

+ * 
This task supports copying either the Rest YAML tests (.yml), or the Rest API specification (.json).

+ * 

+ * Rest API specification: 

+ * When the {@link CopyRestApiPlugin} has been applied this task will automatically copy the Rest API specification
+ * if there are any Rest YAML tests present (either in source, or output) or if `restApi.includeCore` or `restApi.includeXpack` has been
+ * explicitly declared through the 'restResources' extension. 

+ * This task supports copying only a subset of the Rest API specification through the use of the custom extension.

+ * For example:
+ * 
+ * restResources {
+ *   restApi {
+ *     includeXpack 'enrich'
+ *   }
+ * }
+ * 

+ * Will copy any of the the x-pack specs that start with enrich*. The core API specs will also be copied iff the project also has
+ * Rest YAML tests. To help optimize the build cache, it is recommended to explicitly declare which specs your project depends on.
+ * For example:
+ * 
+ * restResources {
+ *   restApi {
+ *     includeCore 'index', 'cat'
+ *     includeXpack 'enrich'
+ *   }
+ * }
+ * 

+ * 

+ * Rest YAML tests : 

+ * When the {@link CopyRestApiPlugin} has been applied this task can copy the Rest YAML tests iff explicitly configured with
+ * `includeCore` or `includeXpack` through the `restResources.restTests` extension.
+ * For example:
+ * 
+ * restResources {
+ *   restTests {
+ *     includeXpack 'graph'
+ *   }
+ * }
+ * 

+ * Will copy any of the the x-pack tests that start with graph.
+ */
+public class CopyRestTestsTask extends DefaultTask {
+    private static final Logger logger = Logging.getLogger(CopyRestTestsTask.class);
+    final ListProperty includeCore = getProject().getObjects().listProperty(String.class);
+    final ListProperty includeXpack = getProject().getObjects().listProperty(String.class);
+
+    Configuration coreConfig;
+    Configuration xpackConfig;
+    String copyTo;
+
+    private final PatternFilterable corePatternSet;
+    private final PatternFilterable xpackPatternSet;
+
+    public CopyRestTestsTask() {
+        // TODO: blow up if internal and requested x-pack
+        corePatternSet = getPatternSetFactory().create();
+        xpackPatternSet = getPatternSetFactory().create();
+    }
+
+    @Inject
+    protected Factory getPatternSetFactory() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Input
+    public ListProperty getIncludeCore() {
+        return includeCore;
+    }
+
+    @Input
+    public ListProperty getIncludeXpack() {
+        return includeXpack;
+    }
+
+    @SkipWhenEmpty
+    @InputFiles
+    public FileTree getInputDir() {
+        xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
+        ConfigurableFileCollection fileCollection = getProject().files(xpackConfig.getAsFileTree().matching(xpackPatternSet));
+        if (BuildParams.isInternal()) {
+            corePatternSet.setIncludes(includeCore.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
+            fileCollection.plus(coreConfig.getAsFileTree().matching(corePatternSet));
+        } else {
+            fileCollection.plus(coreConfig);
+        }
+        // copy tests only if explicitly requested
+        return includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false ? fileCollection.getAsFileTree() : null;
+    }
+
+    @OutputDirectory
+    public File getOutputDir() {
+        return new File(getTestSourceSet().getOutput().getResourcesDir(), copyTo);
+    }
+
+    @TaskAction
+    void copy() {
+        Project project = getProject();
+        // only copy core tests if explicitly instructed
+        if (includeCore.get().isEmpty() == false) {
+            if (BuildParams.isInternal()) {
+                logger.info("Rest tests for project [{}] will be copied to the test resources.", project.getPath());
+                project.copy(c -> {
+                    c.from(coreConfig.getSingleFile());
+                    c.into(getOutputDir());
+                    c.include(corePatternSet.getIncludes());
+                });
+
+            } else {
+                logger.info(
+                    "Rest tests for project [{}] will be copied to the test resources from the published jar (version: [{}]).",
+                    project.getPath(),
+                    VersionProperties.getElasticsearch()
+                );
+                project.copy(c -> {
+                    c.from(project.zipTree(coreConfig.getSingleFile()));
+                    c.into(getTestSourceSet().getOutput().getResourcesDir()); // this ends up as the same dir as outputDir
+                    c.include(includeCore.get().stream().map(prefix -> copyTo + "/" + prefix + "*/**").collect(Collectors.toList()));
+                });
+            }
+        }
+        // only copy x-pack tests if explicitly instructed
+        if (includeXpack.get().isEmpty() == false) {
+            logger.info("X-pack rest tests for project [{}] will be copied to the test resources.", project.getPath());
+            project.copy(c -> {
+                c.from(xpackConfig.getSingleFile());
+                c.into(getOutputDir());
+                c.include(xpackPatternSet.getIncludes());
+            });
+        }
+    }
+
+    private SourceSet getTestSourceSet() {
+        return Boilerplate.getJavaSourceSets(getProject()).findByName("test");
+    }
+
+}

From 2f5739cbf81e8db69b14a3624dea94e3eb4b892d Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Thu, 13 Feb 2020 10:38:28 -0600
Subject: [PATCH 12/23] fix smoke-test-security-with-mustache test

---
 x-pack/qa/smoke-test-security-with-mustache/build.gradle | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/x-pack/qa/smoke-test-security-with-mustache/build.gradle b/x-pack/qa/smoke-test-security-with-mustache/build.gradle
index 748252044c36c..3c38b9ae8f30f 100644
--- a/x-pack/qa/smoke-test-security-with-mustache/build.gradle
+++ b/x-pack/qa/smoke-test-security-with-mustache/build.gradle
@@ -6,6 +6,12 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
+restResources {
+  restApi {
+    includeXpack 'security'
+  }
+}
+
 testClusters.integTest {
   testDistribution = 'DEFAULT'
   setting 'xpack.watcher.enabled', 'false'

From 907c0ddf3ccdcbef03f86f1619be0356c03f9201 Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Thu, 13 Feb 2020 14:52:53 -0600
Subject: [PATCH 13/23] fix some of the x-pack:qa tests

---
 x-pack/qa/build.gradle                                 |  2 --
 .../qa/smoke-test-watcher-with-security/build.gradle   | 10 ++++++++++
 x-pack/qa/smoke-test-watcher/build.gradle              |  6 ++++++
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/x-pack/qa/build.gradle b/x-pack/qa/build.gradle
index 5968a2ce6be71..f933cac8d31cc 100644
--- a/x-pack/qa/build.gradle
+++ b/x-pack/qa/build.gradle
@@ -1,8 +1,6 @@
 // this file must exist so that qa projects are found
 // by the elasticsearch x-plugins include mechanism
 
-import org.elasticsearch.gradle.test.RestIntegTestTask
-
 apply plugin: 'elasticsearch.build'
 test.enabled = false
 
diff --git a/x-pack/qa/smoke-test-watcher-with-security/build.gradle b/x-pack/qa/smoke-test-watcher-with-security/build.gradle
index 236ec27cfa4e0..3485b25fda710 100644
--- a/x-pack/qa/smoke-test-watcher-with-security/build.gradle
+++ b/x-pack/qa/smoke-test-watcher-with-security/build.gradle
@@ -13,6 +13,16 @@ task copyWatcherRestTests(type: Copy) {
   include 'rest-api-spec/test/watcher/**'
 }
 
+restResources {
+  restApi {
+    includeXpack 'watcher', 'security', 'xpack'
+  }
+  restTests {
+    includeXpack 'watcher'
+  }
+}
+
+
 integTest.runner.dependsOn copyWatcherRestTests
 testClusters.integTest {
   testDistribution = 'DEFAULT'
diff --git a/x-pack/qa/smoke-test-watcher/build.gradle b/x-pack/qa/smoke-test-watcher/build.gradle
index b3b638e938343..51d56669b837e 100644
--- a/x-pack/qa/smoke-test-watcher/build.gradle
+++ b/x-pack/qa/smoke-test-watcher/build.gradle
@@ -6,6 +6,12 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
+restResources {
+  restApi {
+    includeXpack 'watcher'
+  }
+}
+
 testClusters.integTest {
   testDistribution = 'DEFAULT'
   setting 'xpack.slm.enabled', 'false'

From 173b494ed2701a97bce4755359692eab70ca935a Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Thu, 13 Feb 2020 15:30:54 -0600
Subject: [PATCH 14/23] fix another x-pack qa test

---
 .../gradle/test/rest/CopyRestApiPlugin.java   | 10 ++--
 .../gradle/test/rest/CopyRestApiTask.java     |  9 +++-
 .../gradle/test/rest/CopyRestTestsTask.java   | 46 ++-----------------
 .../build.gradle                              |  6 +++
 4 files changed, 21 insertions(+), 50 deletions(-)

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java
index 721ac4901911b..3cc62bf14d020 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java
@@ -30,12 +30,10 @@
 //TODO: clean up this doc!
 /**
  * 

- * Gradle plugin to help configure {@link CopyRestApiTask}'s that will copy the artifacts needed for the Rest API spec and YAML tests.
- * 

- * 
Copies the files needed for the Rest YAML tests to the current projects test resources output directory.
- * This is intended to be be used from {@link CopyRestApiPlugin} since the plugin wires up the needed
- * configurations and custom extensions.
+ * Gradle plugin to help configure {@link CopyRestApiTask}'s and {@link CopyRestTestsTask} that copies the artifacts needed for the Rest API
+ * spec and YAML based rest tests.
  * 

+
  * 
This task supports copying either the Rest YAML tests (.yml), or the Rest API specification (.json).

  * 

  * Rest API specification: 

@@ -91,7 +89,6 @@ public void apply(Project project) {
             .register("copyYamlTestsTask", CopyRestTestsTask.class, task -> {
                 task.includeCore.set(extension.restTests.getIncludeCore());
                 task.includeXpack.set(extension.restTests.getIncludeXpack());
-                task.copyTo = "rest-api-spec/test";
                 task.coreConfig = project.getConfigurations().create("restTest");
                 if (BuildParams.isInternal()) {
                     Dependency dependency = project.getDependencies()
@@ -115,7 +112,6 @@ public void apply(Project project) {
             .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> {
                 task.includeCore.set(extension.restApi.getIncludeCore());
                 task.includeXpack.set(extension.restApi.getIncludeXpack());
-                task.copyTo = "rest-api-spec/api";
                 task.dependsOn(copyRestYamlTestTask);
                 task.coreConfig = project.getConfigurations().create("restSpec");
                 if (BuildParams.isInternal()) {
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
index c40b95811cbf4..efe542e3d80f8 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
@@ -46,14 +46,21 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+/**
+ * Copies the files needed for the Rest YAML specs to the current projects test resources output directory.
+ * This is intended to be be used from {@link CopyRestApiPlugin} since the plugin wires up the needed
+ * configurations and custom extensions.
+ * @see CopyRestApiPlugin
+ */
 public class CopyRestApiTask extends DefaultTask {
     private static final Logger logger = Logging.getLogger(CopyRestApiTask.class);
+    private static final String copyTo = "rest-api-spec/api";
     final ListProperty includeCore = getProject().getObjects().listProperty(String.class);
     final ListProperty includeXpack = getProject().getObjects().listProperty(String.class);
 
     Configuration coreConfig;
     Configuration xpackConfig;
-    String copyTo;
+
 
     private final PatternFilterable corePatternSet;
     private final PatternFilterable xpackPatternSet;
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
index c7202f3377742..d4fe69f0533e7 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -44,58 +44,20 @@
 import java.util.stream.Collectors;
 
 /**
- * 
Copies the files needed for the Rest YAML tests to the current projects test resources output directory.
+ * Copies the Rest YAML test to the current projects test resources output directory.
  * This is intended to be be used from {@link CopyRestApiPlugin} since the plugin wires up the needed
  * configurations and custom extensions.
- * 

- * 
This task supports copying either the Rest YAML tests (.yml), or the Rest API specification (.json).

- * 

- * Rest API specification: 

- * When the {@link CopyRestApiPlugin} has been applied this task will automatically copy the Rest API specification
- * if there are any Rest YAML tests present (either in source, or output) or if `restApi.includeCore` or `restApi.includeXpack` has been
- * explicitly declared through the 'restResources' extension. 

- * This task supports copying only a subset of the Rest API specification through the use of the custom extension.

- * For example:
- * 
- * restResources {
- *   restApi {
- *     includeXpack 'enrich'
- *   }
- * }
- * 

- * Will copy any of the the x-pack specs that start with enrich*. The core API specs will also be copied iff the project also has
- * Rest YAML tests. To help optimize the build cache, it is recommended to explicitly declare which specs your project depends on.
- * For example:
- * 
- * restResources {
- *   restApi {
- *     includeCore 'index', 'cat'
- *     includeXpack 'enrich'
- *   }
- * }
- * 

- * 

- * Rest YAML tests : 

- * When the {@link CopyRestApiPlugin} has been applied this task can copy the Rest YAML tests iff explicitly configured with
- * `includeCore` or `includeXpack` through the `restResources.restTests` extension.
- * For example:
- * 
- * restResources {
- *   restTests {
- *     includeXpack 'graph'
- *   }
- * }
- * 

- * Will copy any of the the x-pack tests that start with graph.
+ * @see CopyRestApiPlugin
  */
 public class CopyRestTestsTask extends DefaultTask {
     private static final Logger logger = Logging.getLogger(CopyRestTestsTask.class);
+    private static final String copyTo = "rest-api-spec/test";
     final ListProperty includeCore = getProject().getObjects().listProperty(String.class);
     final ListProperty includeXpack = getProject().getObjects().listProperty(String.class);
 
     Configuration coreConfig;
     Configuration xpackConfig;
-    String copyTo;
+
 
     private final PatternFilterable corePatternSet;
     private final PatternFilterable xpackPatternSet;
diff --git a/x-pack/qa/multi-cluster-search-security/build.gradle b/x-pack/qa/multi-cluster-search-security/build.gradle
index 3f3d03a7e685e..715d8857bac1b 100644
--- a/x-pack/qa/multi-cluster-search-security/build.gradle
+++ b/x-pack/qa/multi-cluster-search-security/build.gradle
@@ -7,6 +7,12 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
+restResources {
+  restApi {
+    includeXpack 'security'
+  }
+}
+
 task 'remote-cluster'(type: RestIntegTestTask) {
   mustRunAfter(precommit)
   runner {

From 6d0aec5938c32aca802b75276d8c867ee12061d3 Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Thu, 13 Feb 2020 16:08:05 -0600
Subject: [PATCH 15/23] fix spotless

---
 .../java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java | 1 -
 .../org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java    | 1 -
 2 files changed, 2 deletions(-)

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
index efe542e3d80f8..51e5f70001a8c 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
@@ -61,7 +61,6 @@ public class CopyRestApiTask extends DefaultTask {
     Configuration coreConfig;
     Configuration xpackConfig;
 
-
     private final PatternFilterable corePatternSet;
     private final PatternFilterable xpackPatternSet;
 
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
index d4fe69f0533e7..b3a71c54015fd 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -58,7 +58,6 @@ public class CopyRestTestsTask extends DefaultTask {
     Configuration coreConfig;
     Configuration xpackConfig;
 
-
     private final PatternFilterable corePatternSet;
     private final PatternFilterable xpackPatternSet;
 

From 6a74f0b20a621cde5528453fa3146f12d414cb94 Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Thu, 13 Feb 2020 17:24:43 -0600
Subject: [PATCH 16/23] another qa xpack test fix

---
 x-pack/qa/multi-cluster-tests-with-security/build.gradle | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/x-pack/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/qa/multi-cluster-tests-with-security/build.gradle
index 74b15cd87a51f..80a560aa42f1b 100644
--- a/x-pack/qa/multi-cluster-tests-with-security/build.gradle
+++ b/x-pack/qa/multi-cluster-tests-with-security/build.gradle
@@ -8,6 +8,12 @@ dependencies {
   testCompile project(':client:rest-high-level')
 }
 
+restResources {
+  restApi {
+    includeXpack 'security'
+  }
+}
+
 task 'remote-cluster'(type: RestIntegTestTask) {
   mustRunAfter(precommit)
   runner {

From a391dd73d667e290005e8b55c7d1475c8bb9be78 Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Fri, 14 Feb 2020 09:27:55 -0600
Subject: [PATCH 17/23] fix another x-pack test (the prior iteration auto added
 all of  x-pack specs, now you have to declare which x-pack specs)

---
 x-pack/qa/multi-cluster-tests-with-security/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/qa/multi-cluster-tests-with-security/build.gradle
index 80a560aa42f1b..b6a3088f250aa 100644
--- a/x-pack/qa/multi-cluster-tests-with-security/build.gradle
+++ b/x-pack/qa/multi-cluster-tests-with-security/build.gradle
@@ -10,7 +10,7 @@ dependencies {
 
 restResources {
   restApi {
-    includeXpack 'security'
+    includeXpack 'security', 'transform'
   }
 }
 

From 9583d15eacde9d5b270d1407d0192ee4518f58ec Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Mon, 24 Feb 2020 12:15:12 -0600
Subject: [PATCH 18/23] clean up

---
 .../gradle/test/rest/CopyRestApiPlugin.java   | 28 +++++------
 .../gradle/test/rest/CopyRestApiTask.java     |  8 ++-
 .../gradle/test/rest/CopyRestTestsTask.java   |  8 ++-
 .../test/rest/RestResourcesExtension.java     | 32 ++++++++++++
 .../gradle/test/rest/RestResourcesSpec.java   | 49 -------------------
 5 files changed, 52 insertions(+), 73 deletions(-)
 delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesSpec.java

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java
index 3cc62bf14d020..acd4e0948cd92 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java
@@ -27,20 +27,16 @@
 
 import java.util.Map;
 
-//TODO: clean up this doc!
 /**
  * 

  * Gradle plugin to help configure {@link CopyRestApiTask}'s and {@link CopyRestTestsTask} that copies the artifacts needed for the Rest API
  * spec and YAML based rest tests.
  * 

-
- * 
This task supports copying either the Rest YAML tests (.yml), or the Rest API specification (.json).

- * 

  * Rest API specification: 

- * When the {@link CopyRestApiPlugin} has been applied this task will automatically copy the Rest API specification
- * if there are any Rest YAML tests present (either in source, or output) or if `restApi.includeCore` or `restApi.includeXpack` has been
- * explicitly declared through the 'restResources' extension. 

- * This task supports copying only a subset of the Rest API specification through the use of the custom extension.

+ * When the {@link CopyRestApiPlugin} has been applied the {@link CopyRestApiTask} will automatically copy the core Rest API specification
+ * if there are any Rest YAML tests present in source, or copied from {@link CopyRestTestsTask} output. X-pack specs must be explicitly
+ * declared to be copied.
+ * 

  * For example:
  * 
  * restResources {
@@ -49,8 +45,9 @@
  *   }
  * }
  * 

- * Will copy any of the the x-pack specs that start with enrich*. The core API specs will also be copied iff the project also has
- * Rest YAML tests. To help optimize the build cache, it is recommended to explicitly declare which specs your project depends on.
+ * Will copy the entire core Rest API specifications (assuming the project has tests) and any of the the X-pack specs starting with enrich*.
+ * It is recommended (but not required) to also explicitly declare which core specs your project depends on to help optimize the caching
+ * behavior.
  * For example:
  * 
  * restResources {
@@ -62,17 +59,21 @@
  * 

  * 

  * Rest YAML tests : 

- * When the {@link CopyRestApiPlugin} has been applied this task can copy the Rest YAML tests iff explicitly configured with
- * `includeCore` or `includeXpack` through the `restResources.restTests` extension.
+ * When the {@link CopyRestApiPlugin} has been applied the {@link CopyRestTestsTask} will copy the Rest YAML tests if explicitly
+ * configured with `includeCore` or `includeXpack` through the `restResources.restTests` extension.
  * For example:
  * 
  * restResources {
+ *  restApi {
+ *      includeXpack 'graph'
+ *   }
  *   restTests {
  *     includeXpack 'graph'
  *   }
  * }
  * 

- * Will copy any of the the x-pack tests that start with graph.
+ * Will copy any of the the x-pack tests that start with graph, and will copy the X-pack graph specification, as well as the full core
+ * Rest API specification.
  *
  * @see CopyRestApiTask
  * @see CopyRestTestsTask
@@ -99,7 +100,6 @@ public void apply(Project project) {
                     dependency = project.getDependencies().project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
                     project.getDependencies().add(task.xpackConfig.getName(), dependency);
                     task.dependsOn(task.xpackConfig);
-
                 } else {
                     Dependency dependency = project.getDependencies()
                         .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
index 51e5f70001a8c..fb31094194058 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
@@ -54,7 +54,7 @@
  */
 public class CopyRestApiTask extends DefaultTask {
     private static final Logger logger = Logging.getLogger(CopyRestApiTask.class);
-    private static final String copyTo = "rest-api-spec/api";
+    private static final String COPY_TO = "rest-api-spec/api";
     final ListProperty includeCore = getProject().getObjects().listProperty(String.class);
     final ListProperty includeXpack = getProject().getObjects().listProperty(String.class);
 
@@ -65,7 +65,6 @@ public class CopyRestApiTask extends DefaultTask {
     private final PatternFilterable xpackPatternSet;
 
     public CopyRestApiTask() {
-        // TODO: blow up if internal and requested x-pack
         corePatternSet = getPatternSetFactory().create();
         xpackPatternSet = getPatternSetFactory().create();
     }
@@ -104,7 +103,7 @@ public FileTree getInputDir() {
 
     @OutputDirectory
     public File getOutputDir() {
-        return new File(getTestSourceSet().getOutput().getResourcesDir(), copyTo);
+        return new File(getTestSourceSet().getOutput().getResourcesDir(), COPY_TO);
     }
 
     @TaskAction
@@ -118,7 +117,6 @@ void copy() {
                 c.into(getOutputDir());
                 c.include(corePatternSet.getIncludes());
             });
-
         } else {
             logger.info(
                 "Rest specs for project [{}] will be copied to the test resources from the published jar (version: [{}]).",
@@ -128,7 +126,7 @@ void copy() {
             project.copy(c -> {
                 c.from(project.zipTree(coreConfig.getSingleFile()));
                 c.into(getTestSourceSet().getOutput().getResourcesDir()); // this ends up as the same dir as outputDir
-                c.include(includeCore.get().stream().map(prefix -> copyTo + "/" + prefix + "*/**").collect(Collectors.toList()));
+                c.include(includeCore.get().stream().map(prefix -> COPY_TO + "/" + prefix + "*/**").collect(Collectors.toList()));
             });
         }
         // only copy x-pack specs if explicitly instructed
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
index b3a71c54015fd..e2791f3ab163b 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -51,7 +51,7 @@
  */
 public class CopyRestTestsTask extends DefaultTask {
     private static final Logger logger = Logging.getLogger(CopyRestTestsTask.class);
-    private static final String copyTo = "rest-api-spec/test";
+    private static final String COPY_TO = "rest-api-spec/test";
     final ListProperty includeCore = getProject().getObjects().listProperty(String.class);
     final ListProperty includeXpack = getProject().getObjects().listProperty(String.class);
 
@@ -62,7 +62,6 @@ public class CopyRestTestsTask extends DefaultTask {
     private final PatternFilterable xpackPatternSet;
 
     public CopyRestTestsTask() {
-        // TODO: blow up if internal and requested x-pack
         corePatternSet = getPatternSetFactory().create();
         xpackPatternSet = getPatternSetFactory().create();
     }
@@ -99,7 +98,7 @@ public FileTree getInputDir() {
 
     @OutputDirectory
     public File getOutputDir() {
-        return new File(getTestSourceSet().getOutput().getResourcesDir(), copyTo);
+        return new File(getTestSourceSet().getOutput().getResourcesDir(), COPY_TO);
     }
 
     @TaskAction
@@ -124,7 +123,7 @@ void copy() {
                 project.copy(c -> {
                     c.from(project.zipTree(coreConfig.getSingleFile()));
                     c.into(getTestSourceSet().getOutput().getResourcesDir()); // this ends up as the same dir as outputDir
-                    c.include(includeCore.get().stream().map(prefix -> copyTo + "/" + prefix + "*/**").collect(Collectors.toList()));
+                    c.include(includeCore.get().stream().map(prefix -> COPY_TO + "/" + prefix + "*/**").collect(Collectors.toList()));
                 });
             }
         }
@@ -142,5 +141,4 @@ void copy() {
     private SourceSet getTestSourceSet() {
         return Boilerplate.getJavaSourceSets(getProject()).findByName("test");
     }
-
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java
index df51af42fe13a..2865963bdb3bd 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java
@@ -18,8 +18,10 @@
  */
 package org.elasticsearch.gradle.test.rest;
 
+import org.elasticsearch.gradle.info.BuildParams;
 import org.gradle.api.Action;
 import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.ListProperty;
 
 import javax.inject.Inject;
 
@@ -44,4 +46,34 @@ void restApi(Action spec) {
     void restTests(Action spec) {
         spec.execute(restTests);
     }
+
+    static class RestResourcesSpec {
+
+        private final ListProperty includeCore;
+        private final ListProperty includeXpack;
+
+        RestResourcesSpec(ObjectFactory objects) {
+            includeCore = objects.listProperty(String.class);
+            includeXpack = objects.listProperty(String.class);
+        }
+
+        public void includeCore(String... include) {
+            this.includeCore.addAll(include);
+        }
+
+        public void includeXpack(String... include) {
+            if (BuildParams.isInternal() == false) {
+                throw new IllegalStateException("Can not include x-pack rest resources from an external build.");
+            }
+            this.includeXpack.addAll(include);
+        }
+
+        public ListProperty getIncludeCore() {
+            return includeCore;
+        }
+
+        public ListProperty getIncludeXpack() {
+            return includeXpack;
+        }
+    }
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesSpec.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesSpec.java
deleted file mode 100644
index 49e7b62731a8b..0000000000000
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesSpec.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.gradle.test.rest;
-
-import org.gradle.api.model.ObjectFactory;
-import org.gradle.api.provider.ListProperty;
-
-public class RestResourcesSpec {
-
-    private final ListProperty includeCore;
-    private final ListProperty includeXpack;
-
-    public RestResourcesSpec(ObjectFactory objects) {
-        includeCore = objects.listProperty(String.class);
-        includeXpack = objects.listProperty(String.class);
-    }
-
-    public void includeCore(String... include) {
-        this.includeCore.addAll(include);
-    }
-
-    public void includeXpack(String... include) {
-        this.includeXpack.addAll(include);
-    }
-
-    public ListProperty getIncludeCore() {
-        return includeCore;
-    }
-
-    public ListProperty getIncludeXpack() {
-        return includeXpack;
-    }
-}

From d86fc58b580a6a4b0095f7e3d0d046ccdf7d0d0e Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Mon, 24 Feb 2020 12:33:02 -0600
Subject: [PATCH 19/23] move logger to debug and remove incidental change

---
 .../org/elasticsearch/gradle/test/rest/CopyRestApiTask.java | 6 +++---
 .../elasticsearch/gradle/test/rest/CopyRestTestsTask.java   | 6 +++---
 .../java/org/elasticsearch/gradle/info/BuildParams.java     | 2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
index fb31094194058..ffb5275721714 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
@@ -111,14 +111,14 @@ void copy() {
         Project project = getProject();
         // always copy the core specs if the task executes
         if (BuildParams.isInternal()) {
-            logger.info("Rest specs for project [{}] will be copied to the test resources.", project.getPath());
+            logger.debug("Rest specs for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
                 c.from(coreConfig.getSingleFile());
                 c.into(getOutputDir());
                 c.include(corePatternSet.getIncludes());
             });
         } else {
-            logger.info(
+            logger.debug(
                 "Rest specs for project [{}] will be copied to the test resources from the published jar (version: [{}]).",
                 project.getPath(),
                 VersionProperties.getElasticsearch()
@@ -131,7 +131,7 @@ void copy() {
         }
         // only copy x-pack specs if explicitly instructed
         if (includeXpack.get().isEmpty() == false) {
-            logger.info("X-pack rest specs for project [{}] will be copied to the test resources.", project.getPath());
+            logger.debug("X-pack rest specs for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
                 c.from(xpackConfig.getSingleFile());
                 c.into(getOutputDir());
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
index e2791f3ab163b..969ec1f4e095f 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -107,7 +107,7 @@ void copy() {
         // only copy core tests if explicitly instructed
         if (includeCore.get().isEmpty() == false) {
             if (BuildParams.isInternal()) {
-                logger.info("Rest tests for project [{}] will be copied to the test resources.", project.getPath());
+                logger.debug("Rest tests for project [{}] will be copied to the test resources.", project.getPath());
                 project.copy(c -> {
                     c.from(coreConfig.getSingleFile());
                     c.into(getOutputDir());
@@ -115,7 +115,7 @@ void copy() {
                 });
 
             } else {
-                logger.info(
+                logger.debug(
                     "Rest tests for project [{}] will be copied to the test resources from the published jar (version: [{}]).",
                     project.getPath(),
                     VersionProperties.getElasticsearch()
@@ -129,7 +129,7 @@ void copy() {
         }
         // only copy x-pack tests if explicitly instructed
         if (includeXpack.get().isEmpty() == false) {
-            logger.info("X-pack rest tests for project [{}] will be copied to the test resources.", project.getPath());
+            logger.debug("X-pack rest tests for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
                 c.from(xpackConfig.getSingleFile());
                 c.into(getOutputDir());
diff --git a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java
index e59bdbe15b61e..a76413d6b47d4 100644
--- a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java
+++ b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java
@@ -147,7 +147,7 @@ private static String propertyName(String methodName) {
     }
 
     public static class MutableBuildParams {
-        public static MutableBuildParams INSTANCE = new MutableBuildParams();
+        private static MutableBuildParams INSTANCE = new MutableBuildParams();
 
         private MutableBuildParams() {}
 

From 852b30e92739a80289ec819243d3f273e7ccf4bf Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Tue, 25 Feb 2020 10:01:44 -0600
Subject: [PATCH 20/23] fix 3rd party config and remove uneeded copy

---
 .../build.gradle                              |  9 --------
 x-pack/qa/third-party/jira/build.gradle       | 23 +++++++++++--------
 x-pack/qa/third-party/pagerduty/build.gradle  |  7 ++++++
 x-pack/qa/third-party/slack/build.gradle      |  7 ++++++
 4 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/x-pack/qa/smoke-test-watcher-with-security/build.gradle b/x-pack/qa/smoke-test-watcher-with-security/build.gradle
index 3485b25fda710..6fbbfa87ed557 100644
--- a/x-pack/qa/smoke-test-watcher-with-security/build.gradle
+++ b/x-pack/qa/smoke-test-watcher-with-security/build.gradle
@@ -6,13 +6,6 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
-// bring in watcher rest test suite
-task copyWatcherRestTests(type: Copy) {
-  into project.sourceSets.test.output.resourcesDir
-  from project(xpackProject('plugin').path).sourceSets.test.resources.srcDirs
-  include 'rest-api-spec/test/watcher/**'
-}
-
 restResources {
   restApi {
     includeXpack 'watcher', 'security', 'xpack'
@@ -22,8 +15,6 @@ restResources {
   }
 }
 
-
-integTest.runner.dependsOn copyWatcherRestTests
 testClusters.integTest {
   testDistribution = 'DEFAULT'
   setting 'xpack.ilm.enabled', 'false'
diff --git a/x-pack/qa/third-party/jira/build.gradle b/x-pack/qa/third-party/jira/build.gradle
index f724d5f1f3674..f2642d7150049 100644
--- a/x-pack/qa/third-party/jira/build.gradle
+++ b/x-pack/qa/third-party/jira/build.gradle
@@ -12,22 +12,17 @@ dependencies {
   testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
 }
 
+restResources {
+  restApi {
+    includeXpack 'watcher'
+  }
+}
 
 String jiraUrl = System.getenv('jira_url')
 String jiraUser = System.getenv('jira_user')
 String jiraPassword = System.getenv('jira_password')
 String jiraProject = System.getenv('jira_project')
 
-testClusters.integTest {
-  setting 'xpack.security.enabled', 'false'
-  setting 'xpack.monitoring.enabled', 'false'
-  setting 'xpack.ml.enabled', 'false'
-  setting 'xpack.license.self_generated.type', 'trial'
-  setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG'
-  setting 'xpack.notification.jira.account.test.issue_defaults.issuetype.name', 'Bug'
-  setting 'xpack.notification.jira.account.test.issue_defaults.labels.0', 'integration-tests'
-}
-
 task cleanJira(type: DefaultTask) {
   doLast {
     List issues = jiraIssues(jiraProject)
@@ -46,6 +41,14 @@ if (!jiraUrl && !jiraUser && !jiraPassword && !jiraProject) {
   testingConventions.enabled = false
 } else {
   testClusters.integTest {
+    testDistribution = 'DEFAULT'
+    setting 'xpack.security.enabled', 'false'
+    setting 'xpack.monitoring.enabled', 'false'
+    setting 'xpack.ml.enabled', 'false'
+    setting 'xpack.license.self_generated.type', 'trial'
+    setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG'
+    setting 'xpack.notification.jira.account.test.issue_defaults.issuetype.name', 'Bug'
+    setting 'xpack.notification.jira.account.test.issue_defaults.labels.0', 'integration-tests'
     setting 'xpack.notification.jira.account.test.issue_defaults.project.key', jiraProject
     keystore 'xpack.notification.jira.account.test.secure_url', jiraUrl
     keystore 'xpack.notification.jira.account.test.secure_user', jiraUser
diff --git a/x-pack/qa/third-party/pagerduty/build.gradle b/x-pack/qa/third-party/pagerduty/build.gradle
index ebee73f9eaaed..c65f04118e0d3 100644
--- a/x-pack/qa/third-party/pagerduty/build.gradle
+++ b/x-pack/qa/third-party/pagerduty/build.gradle
@@ -9,11 +9,18 @@ dependencies {
 
 String pagerDutyServiceKey = System.getenv('pagerduty_service_api_key')
 
+restResources {
+  restApi {
+    includeXpack 'watcher'
+  }
+}
+
 if (!pagerDutyServiceKey) {
   integTest.enabled = false
   testingConventions.enabled = false
 } else {
   testClusters.integTest {
+    testDistribution = 'DEFAULT'
     setting 'xpack.security.enabled', 'false'
     setting 'xpack.monitoring.enabled', 'false'
     setting 'xpack.ml.enabled', 'false'
diff --git a/x-pack/qa/third-party/slack/build.gradle b/x-pack/qa/third-party/slack/build.gradle
index 815d1cb3bbfff..a1b92050482bb 100644
--- a/x-pack/qa/third-party/slack/build.gradle
+++ b/x-pack/qa/third-party/slack/build.gradle
@@ -7,6 +7,12 @@ dependencies {
   testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
 }
 
+restResources {
+  restApi {
+    includeXpack 'watcher'
+  }
+}
+
 String slackUrl = System.getenv('slack_url')
 
 if (!slackUrl) {
@@ -14,6 +20,7 @@ if (!slackUrl) {
   testingConventions.enabled = false
 } else {
   testClusters.integTest {
+    testDistribution = 'DEFAULT'
     setting 'xpack.security.enabled', 'false'
     setting 'xpack.monitoring.enabled', 'false'
     setting 'xpack.ml.enabled', 'false'

From 10700bc2fdcbb146afa6ebe7c2b444fbd321e217 Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Tue, 25 Feb 2020 15:57:32 -0600
Subject: [PATCH 21/23] address review comments

---
 .../gradle/plugin/PluginBuildPlugin.groovy    |  4 ++--
 .../test/StandaloneRestTestPlugin.groovy      |  4 ++--
 .../gradle/test/rest/CopyRestApiTask.java     | 11 ++++-----
 .../gradle/test/rest/CopyRestTestsTask.java   | 11 ++++-----
 ...piPlugin.java => RestResourcesPlugin.java} | 24 ++++++++++---------
 .../elasticsearch.rest-api-copy.properties    |  2 +-
 6 files changed, 28 insertions(+), 28 deletions(-)
 rename buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/{CopyRestApiPlugin.java => RestResourcesPlugin.java} (84%)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
index 1118ba53ec7ee..88060ebc0837c 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
@@ -24,7 +24,7 @@ import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.info.BuildParams
-import org.elasticsearch.gradle.test.rest.CopyRestApiPlugin
+import org.elasticsearch.gradle.test.rest.RestResourcesPlugin
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.testclusters.RunTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
@@ -52,7 +52,7 @@ class PluginBuildPlugin implements Plugin {
     void apply(Project project) {
         project.pluginManager.apply(BuildPlugin)
         project.pluginManager.apply(TestClustersPlugin)
-        project.pluginManager.apply(CopyRestApiPlugin)
+        project.pluginManager.apply(RestResourcesPlugin)
 
         PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project)
         configureDependencies(project)
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
index c75270b8896b7..d668aa4b6b7fa 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
@@ -26,7 +26,7 @@ import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
 import org.elasticsearch.gradle.precommit.PrecommitTasks
-import org.elasticsearch.gradle.test.rest.CopyRestApiPlugin
+import org.elasticsearch.gradle.test.rest.RestResourcesPlugin
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
@@ -76,7 +76,7 @@ class StandaloneRestTestPlugin implements Plugin {
         SourceSetContainer sourceSets = project.extensions.getByType(SourceSetContainer)
         SourceSet testSourceSet = sourceSets.create('test')
         // need to apply plugin after test source sets are created
-        project.pluginManager.apply(CopyRestApiPlugin)
+        project.pluginManager.apply(RestResourcesPlugin)
 
         project.tasks.withType(Test) { Test test ->
             test.testClassesDirs = testSourceSet.output.classesDirs
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
index ffb5275721714..83b62e8d56254 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
@@ -48,12 +48,11 @@
 
 /**
  * Copies the files needed for the Rest YAML specs to the current projects test resources output directory.
- * This is intended to be be used from {@link CopyRestApiPlugin} since the plugin wires up the needed
+ * This is intended to be be used from {@link RestResourcesPlugin} since the plugin wires up the needed
  * configurations and custom extensions.
- * @see CopyRestApiPlugin
+ * @see RestResourcesPlugin
  */
 public class CopyRestApiTask extends DefaultTask {
-    private static final Logger logger = Logging.getLogger(CopyRestApiTask.class);
     private static final String COPY_TO = "rest-api-spec/api";
     final ListProperty includeCore = getProject().getObjects().listProperty(String.class);
     final ListProperty includeXpack = getProject().getObjects().listProperty(String.class);
@@ -111,14 +110,14 @@ void copy() {
         Project project = getProject();
         // always copy the core specs if the task executes
         if (BuildParams.isInternal()) {
-            logger.debug("Rest specs for project [{}] will be copied to the test resources.", project.getPath());
+            getLogger().debug("Rest specs for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
                 c.from(coreConfig.getSingleFile());
                 c.into(getOutputDir());
                 c.include(corePatternSet.getIncludes());
             });
         } else {
-            logger.debug(
+            getLogger().debug(
                 "Rest specs for project [{}] will be copied to the test resources from the published jar (version: [{}]).",
                 project.getPath(),
                 VersionProperties.getElasticsearch()
@@ -131,7 +130,7 @@ void copy() {
         }
         // only copy x-pack specs if explicitly instructed
         if (includeXpack.get().isEmpty() == false) {
-            logger.debug("X-pack rest specs for project [{}] will be copied to the test resources.", project.getPath());
+            getLogger().debug("X-pack rest specs for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
                 c.from(xpackConfig.getSingleFile());
                 c.into(getOutputDir());
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
index 969ec1f4e095f..002836271b025 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -45,12 +45,11 @@
 
 /**
  * Copies the Rest YAML test to the current projects test resources output directory.
- * This is intended to be be used from {@link CopyRestApiPlugin} since the plugin wires up the needed
+ * This is intended to be be used from {@link RestResourcesPlugin} since the plugin wires up the needed
  * configurations and custom extensions.
- * @see CopyRestApiPlugin
+ * @see RestResourcesPlugin
  */
 public class CopyRestTestsTask extends DefaultTask {
-    private static final Logger logger = Logging.getLogger(CopyRestTestsTask.class);
     private static final String COPY_TO = "rest-api-spec/test";
     final ListProperty includeCore = getProject().getObjects().listProperty(String.class);
     final ListProperty includeXpack = getProject().getObjects().listProperty(String.class);
@@ -107,7 +106,7 @@ void copy() {
         // only copy core tests if explicitly instructed
         if (includeCore.get().isEmpty() == false) {
             if (BuildParams.isInternal()) {
-                logger.debug("Rest tests for project [{}] will be copied to the test resources.", project.getPath());
+                getLogger().debug("Rest tests for project [{}] will be copied to the test resources.", project.getPath());
                 project.copy(c -> {
                     c.from(coreConfig.getSingleFile());
                     c.into(getOutputDir());
@@ -115,7 +114,7 @@ void copy() {
                 });
 
             } else {
-                logger.debug(
+                getLogger().debug(
                     "Rest tests for project [{}] will be copied to the test resources from the published jar (version: [{}]).",
                     project.getPath(),
                     VersionProperties.getElasticsearch()
@@ -129,7 +128,7 @@ void copy() {
         }
         // only copy x-pack tests if explicitly instructed
         if (includeXpack.get().isEmpty() == false) {
-            logger.debug("X-pack rest tests for project [{}] will be copied to the test resources.", project.getPath());
+            getLogger().debug("X-pack rest tests for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
                 c.from(xpackConfig.getSingleFile());
                 c.into(getOutputDir());
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
similarity index 84%
rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java
rename to buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
index acd4e0948cd92..a512b9b1fc025 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
@@ -33,7 +33,7 @@
  * spec and YAML based rest tests.
  * 

  * Rest API specification: 

- * When the {@link CopyRestApiPlugin} has been applied the {@link CopyRestApiTask} will automatically copy the core Rest API specification
+ * When the {@link RestResourcesPlugin} has been applied the {@link CopyRestApiTask} will automatically copy the core Rest API specification
  * if there are any Rest YAML tests present in source, or copied from {@link CopyRestTestsTask} output. X-pack specs must be explicitly
  * declared to be copied.
  * 

@@ -59,7 +59,7 @@
  *
*
* Rest YAML tests :
- * When the {@link CopyRestApiPlugin} has been applied the {@link CopyRestTestsTask} will copy the Rest YAML tests if explicitly + * When the {@link RestResourcesPlugin} has been applied the {@link CopyRestTestsTask} will copy the Rest YAML tests if explicitly * configured with `includeCore` or `includeXpack` through the `restResources.restTests` extension. * For example: * 
@@ -78,7 +78,7 @@
  * @see CopyRestApiTask
  * @see CopyRestTestsTask
  */
-public class CopyRestApiPlugin implements Plugin {
+public class RestResourcesPlugin implements Plugin {
 
     private static final String EXTENSION_NAME = "restResources";
 
@@ -92,13 +92,14 @@ public void apply(Project project) {
                 task.includeXpack.set(extension.restTests.getIncludeXpack());
                 task.coreConfig = project.getConfigurations().create("restTest");
                 if (BuildParams.isInternal()) {
-                    Dependency dependency = project.getDependencies()
+                    Dependency restTestdependency = project.getDependencies()
                         .project(Map.of("path", ":rest-api-spec", "configuration", "restTests"));
-                    project.getDependencies().add(task.coreConfig.getName(), dependency);
+                    project.getDependencies().add(task.coreConfig.getName(), restTestdependency);
 
                     task.xpackConfig = project.getConfigurations().create("restXpackTest");
-                    dependency = project.getDependencies().project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
-                    project.getDependencies().add(task.xpackConfig.getName(), dependency);
+                    Dependency restXPackTestdependency = project.getDependencies()
+                        .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
+                    project.getDependencies().add(task.xpackConfig.getName(), restXPackTestdependency);
                     task.dependsOn(task.xpackConfig);
                 } else {
                     Dependency dependency = project.getDependencies()
@@ -115,13 +116,14 @@ public void apply(Project project) {
                 task.dependsOn(copyRestYamlTestTask);
                 task.coreConfig = project.getConfigurations().create("restSpec");
                 if (BuildParams.isInternal()) {
-                    Dependency dependency = project.getDependencies()
+                    Dependency restSpecDependency = project.getDependencies()
                         .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs"));
-                    project.getDependencies().add(task.coreConfig.getName(), dependency);
+                    project.getDependencies().add(task.coreConfig.getName(), restSpecDependency);
 
                     task.xpackConfig = project.getConfigurations().create("restXpackSpec");
-                    dependency = project.getDependencies().project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs"));
-                    project.getDependencies().add(task.xpackConfig.getName(), dependency);
+                    Dependency restXpackSpecDependency = project.getDependencies()
+                        .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs"));
+                    project.getDependencies().add(task.xpackConfig.getName(), restXpackSpecDependency);
                     task.dependsOn(task.xpackConfig);
                 } else {
                     Dependency dependency = project.getDependencies()
diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-copy.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-copy.properties
index e2fc92603b519..af2d3e866ea05 100644
--- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-copy.properties
+++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-copy.properties
@@ -17,4 +17,4 @@
 # under the License.
 #
 
-implementation-class=org.elasticsearch.gradle.test.rest.CopyRestApiPlugin
+implementation-class=org.elasticsearch.gradle.test.rest.RestResourcesPlugin

From bb841614f16e2cf18e301ffc449f0e3478b356fa Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Tue, 25 Feb 2020 16:11:49 -0600
Subject: [PATCH 22/23] fix checkStyle

---
 .../org/elasticsearch/gradle/test/rest/CopyRestApiTask.java     | 2 --
 .../org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java   | 2 --
 2 files changed, 4 deletions(-)

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
index 83b62e8d56254..6e66df54dc48d 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
@@ -26,8 +26,6 @@
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.FileTree;
-import org.gradle.api.logging.Logger;
-import org.gradle.api.logging.Logging;
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
index 002836271b025..2fd5a207482a4 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -26,8 +26,6 @@
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.FileTree;
-import org.gradle.api.logging.Logger;
-import org.gradle.api.logging.Logging;
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;

From 9e6e2aa211e25ebfdb858315fb4d596783f9a1cf Mon Sep 17 00:00:00 2001
From: Jake Landis 
Date: Tue, 25 Feb 2020 17:45:57 -0600
Subject: [PATCH 23/23] fix plugin name

---
 ...-copy.properties => elasticsearch.rest-resources.properties} | 0
 client/rest-high-level/build.gradle                             | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename buildSrc/src/main/resources/META-INF/gradle-plugins/{elasticsearch.rest-api-copy.properties => elasticsearch.rest-resources.properties} (100%)

diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-copy.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-resources.properties
similarity index 100%
rename from buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-copy.properties
rename to buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-resources.properties
diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
index 894aa79eb2f31..fabf6e70eacf3 100644
--- a/client/rest-high-level/build.gradle
+++ b/client/rest-high-level/build.gradle
@@ -25,7 +25,7 @@ apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
 apply plugin: 'com.github.johnrengelman.shadow'
-apply plugin: 'elasticsearch.rest-api-copy'
+apply plugin: 'elasticsearch.rest-resources'
 
 group = 'org.elasticsearch.client'
 archivesBaseName = 'elasticsearch-rest-high-level-client'