Smarter copying of the rest specs and tests

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -24,6 +24,7 @@ import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.info.BuildParams
+import org.elasticsearch.gradle.test.RestApiSpecForTestingPlugin
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.testclusters.RunTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
@@ -51,6 +52,7 @@ class PluginBuildPlugin implements Plugin<Project> {
     void apply(Project project) {
         project.pluginManager.apply(BuildPlugin)
         project.pluginManager.apply(TestClustersPlugin)
+        project.pluginManager.apply(RestApiSpecForTestingPlugin)
 
         PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project)
         configureDependencies(project)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -18,29 +18,19 @@
  */
 package org.elasticsearch.gradle.test
 
-import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.RestTestRunnerTask
-import org.elasticsearch.gradle.tool.Boilerplate
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
-import org.gradle.api.file.FileCopyDetails
-import org.gradle.api.tasks.Copy
-import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.testing.Test
-import org.gradle.plugins.ide.idea.IdeaPlugin
+
 /**
  * A wrapper task around setting up a cluster and running rest tests.
  */
 class RestIntegTestTask extends DefaultTask {
 
     protected Test runner
 
-    /** Flag indicating whether the rest tests in the rest spec should be run. */
-    @Input
-    Boolean includePackaged = false
-
     RestIntegTestTask() {
         runner = project.tasks.create("${name}Runner", RestTestRunnerTask.class)
         super.dependsOn(runner)
@@ -69,10 +59,6 @@ class RestIntegTestTask extends DefaultTask {
             runner.systemProperty('test.clustername', System.getProperty("tests.clustername"))
         }
 
-        // copy the rest spec/tests onto the test classpath
-        Copy copyRestSpec = createCopyRestSpecTask()
-        project.sourceSets.test.output.builtBy(copyRestSpec)
-
         // this must run after all projects have been configured, so we know any project
         // references can be accessed as a fully configured
         project.gradle.projectsEvaluated {
@@ -83,12 +69,6 @@ class RestIntegTestTask extends DefaultTask {
         }
     }
 
-    /** Sets the includePackaged property */
-    public void includePackaged(boolean include) {
-        includePackaged = include
-    }
-
-
     @Override
     public Task dependsOn(Object... dependencies) {
         runner.dependsOn(dependencies)
@@ -114,37 +94,4 @@ class RestIntegTestTask extends DefaultTask {
         project.tasks.getByName("${name}Runner").configure(configure)
     }
 
-    Copy createCopyRestSpecTask() {
-        Boilerplate.maybeCreate(project.configurations, 'restSpec') {
-            project.dependencies.add(
-                    'restSpec',
-                    BuildParams.internal ? project.project(':rest-api-spec') :
-                            "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
-            )
-        }
-
-        return Boilerplate.maybeCreate(project.tasks, 'copyRestSpec', Copy) { Copy copy ->
-            copy.dependsOn project.configurations.restSpec
-            copy.into(project.sourceSets.test.output.resourcesDir)
-            copy.from({ project.zipTree(project.configurations.restSpec.singleFile) }) {
-                includeEmptyDirs = false
-                include 'rest-api-spec/**'
-                filesMatching('rest-api-spec/test/**') { FileCopyDetails details ->
-                    if (includePackaged == false) {
-                        details.exclude()
-                    }
-                }
-            }
-
-            if (project.plugins.hasPlugin(IdeaPlugin)) {
-                project.idea {
-                    module {
-                        if (scopes.TEST != null) {
-                            scopes.TEST.plus.add(project.configurations.restSpec)
-                        }
-                    }
-                }
-            }
-        }
-    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -74,6 +74,8 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
         // only setup tests to build
         SourceSetContainer sourceSets = project.extensions.getByType(SourceSetContainer)
         SourceSet testSourceSet = sourceSets.create('test')
+        // need to apply plugin after test source sets are created
+        project.pluginManager.apply(RestApiSpecForTestingPlugin)
 
         project.tasks.withType(Test) { Test test ->
             test.testClassesDirs = testSourceSet.output.classesDirs

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java

@@ -250,7 +250,7 @@ public void doCheck() throws IOException {
             Files.write(getSuccessMarker().toPath(), new byte[] {}, StandardOpenOption.CREATE);
         } else {
             getLogger().error(problems);
-            throw new IllegalStateException("Testing conventions are not honored");
+            throw new IllegalStateException(String.format("Testing conventions [%s] are not honored", problems));
         }
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecExtension.java

@@ -0,0 +1,70 @@
+package org.elasticsearch.gradle.test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class RestApiSpecExtension {
+    private List<String> includesCoreSpec = new ArrayList<>();
+    private List<String> includesCoreTests = new ArrayList<>();
+    private List<String> includesXpackSpec = new ArrayList<>();
+    private List<String> includesXpackTests = new ArrayList<>();
+    private boolean alwaysCopySpec = false;
+    private boolean copyTests = false;
+    private boolean copyXpackTests = false;
+
+    public void includeCoreSpec(String include) {
+        includesCoreSpec.add(include);
+    }
+
+    public List<String> getIncludesCoreSpec() {
+        return includesCoreSpec;
+    }
+
+    public void includeCoreTests(String include) {
+        includesCoreTests.add(include);
+    }
+
+    public List<String> getIncludesCoreTests() {
+        return includesCoreTests;
+    }
+
+    public void includeXpackSpec(String include) {
+        includesXpackSpec.add(include);
+    }
+
+    public List<String> getIncludesXpackSpec() {
+        return includesXpackSpec;
+    }
+
+    public void copyCoreTests(boolean copyTests) {
+        this.copyTests = copyTests;
+    }
+
+    public boolean shouldCopyCoreTests() {
+        return copyTests;
+    }
+
+    public void includeXpackTests(String include) {
+        includesXpackTests.add(include);
+    }
+
+    public List<String> getIncludesXpackTests() {
+        return includesXpackTests;
+    }
+
+    public void copyXpackTests(boolean copyTests) {
+        this.copyXpackTests = copyTests;
+    }
+
+    public boolean shouldCopyXpackTests() {
+        return copyXpackTests;
+    }
+
+    public boolean shouldAlwaysCopySpec() {
+        return alwaysCopySpec;
+    }
+
+    public void alwaysCopySpec(boolean alwaysCopySpec) {
+        this.alwaysCopySpec = alwaysCopySpec;
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/test/RestApiSpecForTestingPlugin.java

@@ -0,0 +1,141 @@
+package org.elasticsearch.gradle.test;
+
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.info.BuildParams;
+import org.elasticsearch.gradle.tool.Boilerplate;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.tasks.Copy;
+import org.gradle.api.tasks.SourceSet;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+//TODO: change logger.info to logger.debug
+public class RestApiSpecForTestingPlugin implements Plugin<Project> {
+
+    private static final Logger logger = Logging.getLogger(RestApiSpecForTestingPlugin.class);
+    private static final String EXTENSION_NAME = "restApiSpec";
+    private static final String apiDir = "rest-api-spec/api";
+    private static final String testDir = "rest-api-spec/test";
+
+    @Override
+    public void apply(Project project) {
+
+        RestApiSpecExtension extension = project.getExtensions().create(EXTENSION_NAME, RestApiSpecExtension.class);
+        // need to defer to after evaluation to allow the custom extension to be populated
+        project.afterEvaluate(p -> {
+            try {
+                // copy tests
+                boolean hasCopiedTests = false;
+                if (extension.shouldCopyCoreTests()) {
+                    Configuration coreTestConfig = project.getConfigurations().create("copyRestSpecCoreTests");
+                    Dependency coreTestDep = project.getDependencies()
+                        .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecCoreTests"));
+                    logger.info("Rest specs tests for project [{}] will be copied to the test resources.", project.getPath());
+                    createCopyTask(project, coreTestConfig, coreTestDep, extension.getIncludesCoreTests(), testDir, false);
+                    hasCopiedTests = true;
+                }
+                if (extension.shouldCopyXpackTests()) {
+                    Configuration xpackSpecTestsConfig = project.getConfigurations().create("copyRestSpecXpackTests");
+                    Dependency xpackTestsDep = project.getDependencies()
+                        .project(Map.of("path", ":x-pack:plugin", "configuration", "restSpecXpackTests"));
+                    logger.info("Rest specs x-pack tests for project [{}] will be copied to the test resources.", project.getPath());
+                    createCopyTask(project, xpackSpecTestsConfig, xpackTestsDep, extension.getIncludesXpackTests(), testDir, false);
+                    hasCopiedTests = true;
+                }
+                // copy specs
+                if (projectHasRestTests(project) || hasCopiedTests || extension.shouldAlwaysCopySpec()) {
+                    Configuration coreSpecConfig = project.getConfigurations().create("restSpec"); // name chosen for passivity
+                    Dependency coreSpecDep = project.getDependencies()
+                        .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecCore"));
+
+                    if (BuildParams.isInternal()) {
+                        // source the specs from this project - this is the path for Elasticsearch builds
+                        if (project.getPath().startsWith(":x-pack")) {
+                            Configuration xpackSpecConfig = project.getConfigurations().create("copyRestSpecXpack");
+                            Dependency xpackSpecDep = project.getDependencies()
+                                .project(Map.of("path", ":x-pack:plugin", "configuration", "restSpecXpack"));
+                            logger.info("X-pack rest specs for project [{}] will be copied to the test resources.", project.getPath());
+                            createCopyTask(project, xpackSpecConfig, xpackSpecDep, extension.getIncludesXpackSpec(), apiDir, false);
+                        }
+
+                        logger.info("Rest specs for project [{}] will be copied to the test resources.", project.getPath());
+                        createCopyTask(project, coreSpecConfig, coreSpecDep, extension.getIncludesCoreSpec(), apiDir, false);
+
+                    } else {
+                        // source the specs from the published jar - this is the path plugin developers
+                        logger.info(
+                            "Rest specs for project [{}] will be copied to the test resources from the published jar (version: [{}]).",
+                            project.getPath(),
+                            VersionProperties.getElasticsearch()
+                        );
+                        Dependency coreSpecFromJarDep = project.getDependencies()
+                            .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
+
+                        createCopyTask(project, coreSpecConfig, coreSpecFromJarDep, extension.getIncludesCoreSpec(), "", true);
+                    }
+                } else {
+                    logger.info("Rest specs will be ignored for project [{}] since there are no REST tests", project.getPath());
+                    return;
+                }
+            } catch (Exception e) {
+                throw new IllegalStateException("Error configuring the rest-api-spec-for-testing plugin. This is likely a bug.", e);
+            }
+        });
+    }
+
+    private void createCopyTask(Project project, Configuration config, Dependency dep, List<String> includes, String into, boolean isJar) {
+        project.getDependencies().add(config.getName(), dep);
+        Copy copyTask = project.getTasks().create(config.getName() + "Task", Copy.class, copy -> {
+            copy.from(isJar ? project.zipTree(config.getSingleFile()) : config.getSingleFile());
+            copy.into(new File(getTestSourceSet(project).getOutput().getResourcesDir(), into));
+            if (includes.isEmpty()) {
+                copy.include(isJar ? apiDir + "/**" : "*/**");
+            } else {
+                includes.forEach(s -> copy.include(isJar ? apiDir + "/" + s + "*/**" : s + "*/**"));
+            }
+            copy.setIncludeEmptyDirs(false);
+        });
+        copyTask.dependsOn(config);
+        project.getTasks().getByName("processTestResources").dependsOn(copyTask);
+    }
+
+    private boolean projectHasRestTests(Project project) throws IOException {
+        File testResourceDir = getTestResourceDir(project);
+        if (testResourceDir == null || new File(testResourceDir, "rest-api-spec/test").exists() == false) {
+            return false;
+        }
+        return Files.walk(testResourceDir.toPath().resolve("rest-api-spec/test")).anyMatch(p -> p.getFileName().toString().endsWith("yml"));
+    }
+
+    private File getTestResourceDir(Project project) {
+        SourceSet testSources = getTestSourceSet(project);
+        if (testSources == null) {
+            return null;
+        }
+        Set<File> resourceDir = testSources.getResources()
+            .getSrcDirs()
+            .stream()
+            .filter(f -> f.isDirectory() && f.getParentFile().getName().equals("test") && f.getName().equals("resources"))
+            .collect(Collectors.toSet());
+        assert resourceDir.size() <= 1;
+        if (resourceDir.size() == 0) {
+            return null;
+        }
+        return resourceDir.iterator().next();
+    }
+
+    private SourceSet getTestSourceSet(Project project) {
+        return Boilerplate.getJavaSourceSets(project).findByName("test");
+    }
+}

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-api-spec-for-testing.properties

@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.test.RestApiSpecForTestingPlugin

buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java

@@ -147,7 +147,7 @@ private static String propertyName(String methodName) {
     }
 
     public static class MutableBuildParams {
-        private static MutableBuildParams INSTANCE = new MutableBuildParams();
+        public static MutableBuildParams INSTANCE = new MutableBuildParams();
 
         private MutableBuildParams() {}