Smarter copying of the rest specs and tests

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -24,6 +24,7 @@ import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.info.BuildParams
+import org.elasticsearch.gradle.test.rest.CopyRestApiPlugin
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.testclusters.RunTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
@@ -51,6 +52,7 @@ class PluginBuildPlugin implements Plugin<Project> {
     void apply(Project project) {
         project.pluginManager.apply(BuildPlugin)
         project.pluginManager.apply(TestClustersPlugin)
+        project.pluginManager.apply(CopyRestApiPlugin)
 
         PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project)
         configureDependencies(project)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -18,29 +18,19 @@
  */
 package org.elasticsearch.gradle.test
 
-import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.RestTestRunnerTask
-import org.elasticsearch.gradle.tool.Boilerplate
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
-import org.gradle.api.file.FileCopyDetails
-import org.gradle.api.tasks.Copy
-import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.testing.Test
-import org.gradle.plugins.ide.idea.IdeaPlugin
+
 /**
  * A wrapper task around setting up a cluster and running rest tests.
  */
 class RestIntegTestTask extends DefaultTask {
 
     protected Test runner
 
-    /** Flag indicating whether the rest tests in the rest spec should be run. */
-    @Input
-    Boolean includePackaged = false
-
     RestIntegTestTask() {
         runner = project.tasks.create("${name}Runner", RestTestRunnerTask.class)
         super.dependsOn(runner)
@@ -69,10 +59,6 @@ class RestIntegTestTask extends DefaultTask {
             runner.systemProperty('test.clustername', System.getProperty("tests.clustername"))
         }
 
-        // copy the rest spec/tests onto the test classpath
-        Copy copyRestSpec = createCopyRestSpecTask()
-        project.sourceSets.test.output.builtBy(copyRestSpec)
-
         // this must run after all projects have been configured, so we know any project
         // references can be accessed as a fully configured
         project.gradle.projectsEvaluated {
@@ -83,12 +69,6 @@ class RestIntegTestTask extends DefaultTask {
         }
     }
 
-    /** Sets the includePackaged property */
-    public void includePackaged(boolean include) {
-        includePackaged = include
-    }
-
-
     @Override
     public Task dependsOn(Object... dependencies) {
         runner.dependsOn(dependencies)
@@ -114,37 +94,4 @@ class RestIntegTestTask extends DefaultTask {
         project.tasks.getByName("${name}Runner").configure(configure)
     }
 
-    Copy createCopyRestSpecTask() {
-        Boilerplate.maybeCreate(project.configurations, 'restSpec') {
-            project.dependencies.add(
-                    'restSpec',
-                    BuildParams.internal ? project.project(':rest-api-spec') :
-                            "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
-            )
-        }
-
-        return Boilerplate.maybeCreate(project.tasks, 'copyRestSpec', Copy) { Copy copy ->
-            copy.dependsOn project.configurations.restSpec
-            copy.into(project.sourceSets.test.output.resourcesDir)
-            copy.from({ project.zipTree(project.configurations.restSpec.singleFile) }) {
-                includeEmptyDirs = false
-                include 'rest-api-spec/**'
-                filesMatching('rest-api-spec/test/**') { FileCopyDetails details ->
-                    if (includePackaged == false) {
-                        details.exclude()
-                    }
-                }
-            }
-
-            if (project.plugins.hasPlugin(IdeaPlugin)) {
-                project.idea {
-                    module {
-                        if (scopes.TEST != null) {
-                            scopes.TEST.plus.add(project.configurations.restSpec)
-                        }
-                    }
-                }
-            }
-        }
-    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -26,9 +26,9 @@ import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
 import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.elasticsearch.gradle.test.rest.CopyRestApiPlugin
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
 import org.gradle.api.InvalidUserDataException
-import org.gradle.api.JavaVersion
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
@@ -42,6 +42,7 @@ import org.gradle.api.tasks.compile.JavaCompile
 import org.gradle.api.tasks.testing.Test
 import org.gradle.plugins.ide.eclipse.model.EclipseModel
 import org.gradle.plugins.ide.idea.model.IdeaModel
+
 /**
  * Configures the build to compile tests against Elasticsearch's test framework
  * and run REST tests. Use BuildPlugin if you want to build main code as well
@@ -74,6 +75,8 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
         // only setup tests to build
         SourceSetContainer sourceSets = project.extensions.getByType(SourceSetContainer)
         SourceSet testSourceSet = sourceSets.create('test')
+        // need to apply plugin after test source sets are created
+        project.pluginManager.apply(CopyRestApiPlugin)
 
         project.tasks.withType(Test) { Test test ->
             test.testClassesDirs = testSourceSet.output.classesDirs

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java

@@ -250,7 +250,7 @@ public void doCheck() throws IOException {
             Files.write(getSuccessMarker().toPath(), new byte[] {}, StandardOpenOption.CREATE);
         } else {
             getLogger().error(problems);
-            throw new IllegalStateException("Testing conventions are not honored");
+            throw new IllegalStateException(String.format("Testing conventions [%s] are not honored", problems));
         }
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiExtension.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.test.rest;
+
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.ListProperty;
+
+/**
+ * Custom extension to configure the {@link CopyRestApiTask}
+ */
+public class CopyRestApiExtension {
+
+    private final ListProperty<String> includeCore;
+    private final ListProperty<String> includeXpack;
+
+    public CopyRestApiExtension(ObjectFactory objects) {
+        includeCore = objects.listProperty(String.class);
+        includeXpack = objects.listProperty(String.class);
+    }
+
+    public void includeCore(String... include) {
+        this.includeCore.addAll(include);
+    }
+
+    public void includeXpack(String... include) {
+        this.includeXpack.addAll(include);
+    }
+
+    public ListProperty<String> getIncludeCore() {
+        return includeCore;
+    }
+
+    public ListProperty<String> getIncludeXpack() {
+        return includeXpack;
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.test.rest;
+
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.info.BuildParams;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.provider.Provider;
+
+import java.util.Map;
+
+/**
+ * Gradle plugin to help configure {@link CopyRestApiTask}'s that will copy the artifacts needed for the Rest API spec and YAML tests.
+ * @see CopyRestApiTask
+ */
+public class CopyRestApiPlugin implements Plugin<Project> {
+
+    private static final String COPY_SPEC_EXTENSION_NAME = "copyRestApiSpecs";
+    private static final String COPY_TEST_EXTENSION_NAME = "copyYamlTests";
+
+    @Override
+    public void apply(Project project) {
+        CopyRestApiExtension copySpecExtension = project.getExtensions().create(COPY_SPEC_EXTENSION_NAME, CopyRestApiExtension.class);
+        CopyRestApiExtension copyTestExtension = project.getExtensions().create(COPY_TEST_EXTENSION_NAME, CopyRestApiExtension.class);
+
+        Provider<CopyRestApiTask> copyRestYamlTestTask = project.getTasks().register("copyYamlTestsTask", CopyRestApiTask.class, task -> {
+            task.includeCore.set(copyTestExtension.getIncludeCore());
+            task.includeXpack.set(copyTestExtension.getIncludeXpack());
+            task.copyTo = "rest-api-spec/test";
+            task.coreConfig = project.getConfigurations().create("restTest");
+            if (BuildParams.isInternal()) {
+                Dependency dependency = project.getDependencies().project(Map.of("path", ":rest-api-spec", "configuration", "restTests"));
+                project.getDependencies().add(task.coreConfig.getName(), dependency);
+            } else {
+                Dependency dependency = project.getDependencies()
+                    .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
+                project.getDependencies().add(task.coreConfig.getName(), dependency);
+            }
+            task.dependsOn(task.coreConfig);
+
+            task.xpackConfig = project.getConfigurations().create("restXpackTest");
+            Dependency dependency = project.getDependencies().project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
+            project.getDependencies().add(task.xpackConfig.getName(), dependency);
+            task.dependsOn(task.xpackConfig);
+        });
+
+        Provider<CopyRestApiTask> copyRestYamlSpecTask = project.getTasks()
+            .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> {
+                task.includeCore.set(copySpecExtension.getIncludeCore());
+                task.includeXpack.set(copySpecExtension.getIncludeXpack());
+                task.copyTo = "rest-api-spec/api";
+                task.dependsOn(copyRestYamlTestTask);
+                task.coreConfig = project.getConfigurations().create("restSpec");
+                if (BuildParams.isInternal()) {
+                    Dependency dependency = project.getDependencies()
+                        .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs"));
+                    project.getDependencies().add(task.coreConfig.getName(), dependency);
+                } else {
+                    Dependency dependency = project.getDependencies()
+                        .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
+                    project.getDependencies().add(task.coreConfig.getName(), dependency);
+                }
+                task.dependsOn(task.coreConfig);
+
+                task.xpackConfig = project.getConfigurations().create("restXpackSpec");
+                Dependency dependency = project.getDependencies()
+                    .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs"));
+                project.getDependencies().add(task.xpackConfig.getName(), dependency);
+                task.dependsOn(task.xpackConfig);
+            });
+
+        project.getTasks().named("processTestResources").configure(t -> t.dependsOn(copyRestYamlSpecTask));
+    }
+}