diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
index 9e9d1e0d9e795..4c86b86b56f5b 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java
@@ -114,7 +114,7 @@ void copy() {
             if (BuildParams.isInternal()) {
                 getLogger().debug("Rest tests for project [{}] will be copied to the test resources.", project.getPath());
                 project.copy(c -> {
-                    c.from(coreConfig.getSingleFile());
+                    c.from(coreConfig.getAsFileTree());
                     c.into(getOutputDir());
                     c.include(corePatternSet.getIncludes());
                 });
@@ -138,7 +138,7 @@ void copy() {
         if (includeXpack.get().isEmpty() == false) {
             getLogger().debug("X-pack rest tests for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
-                c.from(xpackConfig.getSingleFile());
+                c.from(xpackConfig.getAsFileTree());
                 c.into(getOutputDir());
                 c.include(xpackPatternSet.getIncludes());
             });
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
index a512b9b1fc025..57ee2c1f9037f 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java
@@ -86,21 +86,27 @@ public class RestResourcesPlugin implements Plugin<Project> {
     public void apply(Project project) {
         RestResourcesExtension extension = project.getExtensions().create(EXTENSION_NAME, RestResourcesExtension.class);
 
+        // tests
         Provider<CopyRestTestsTask> copyRestYamlTestTask = project.getTasks()
             .register("copyYamlTestsTask", CopyRestTestsTask.class, task -> {
                 task.includeCore.set(extension.restTests.getIncludeCore());
                 task.includeXpack.set(extension.restTests.getIncludeXpack());
-                task.coreConfig = project.getConfigurations().create("restTest");
+                task.coreConfig = project.getConfigurations().maybeCreate("restTests");
                 if (BuildParams.isInternal()) {
+                    // core
                     Dependency restTestdependency = project.getDependencies()
                         .project(Map.of("path", ":rest-api-spec", "configuration", "restTests"));
                     project.getDependencies().add(task.coreConfig.getName(), restTestdependency);
-
-                    task.xpackConfig = project.getConfigurations().create("restXpackTest");
+                    // x-pack
+                    task.xpackConfig = project.getConfigurations().maybeCreate("restXpackTests");
                     Dependency restXPackTestdependency = project.getDependencies()
                         .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
                     project.getDependencies().add(task.xpackConfig.getName(), restXPackTestdependency);
                     task.dependsOn(task.xpackConfig);
+                    // watcher
+                    Dependency restWatcherTests = project.getDependencies()
+                        .project(Map.of("path", ":x-pack:plugin:watcher:qa:rest", "configuration", "restXpackTests"));
+                    project.getDependencies().add(task.xpackConfig.getName(), restWatcherTests);
                 } else {
                     Dependency dependency = project.getDependencies()
                         .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
@@ -109,18 +115,19 @@ public void apply(Project project) {
                 task.dependsOn(task.coreConfig);
             });
 
+        // api
         Provider<CopyRestApiTask> copyRestYamlSpecTask = project.getTasks()
             .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> {
                 task.includeCore.set(extension.restApi.getIncludeCore());
                 task.includeXpack.set(extension.restApi.getIncludeXpack());
                 task.dependsOn(copyRestYamlTestTask);
-                task.coreConfig = project.getConfigurations().create("restSpec");
+                task.coreConfig = project.getConfigurations().maybeCreate("restSpecs");
                 if (BuildParams.isInternal()) {
                     Dependency restSpecDependency = project.getDependencies()
                         .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs"));
                     project.getDependencies().add(task.coreConfig.getName(), restSpecDependency);
 
-                    task.xpackConfig = project.getConfigurations().create("restXpackSpec");
+                    task.xpackConfig = project.getConfigurations().maybeCreate("restXpackSpecs");
                     Dependency restXpackSpecDependency = project.getDependencies()
                         .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs"));
                     project.getDependencies().add(task.xpackConfig.getName(), restXpackSpecDependency);
diff --git a/x-pack/plugin/watcher/qa/rest/build.gradle b/x-pack/plugin/watcher/qa/rest/build.gradle
index df648262e9973..4fb1de7f6cbed 100644
--- a/x-pack/plugin/watcher/qa/rest/build.gradle
+++ b/x-pack/plugin/watcher/qa/rest/build.gradle
@@ -8,6 +8,7 @@ dependencies {
 
 configurations {
   testArtifacts.extendsFrom testRuntime
+  restXpackTests
 }
 
 task testJar(type: Jar) {
@@ -17,6 +18,7 @@ task testJar(type: Jar) {
 
 artifacts {
   testArtifacts testJar
+  restXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test"))
 }
 
 restResources {
diff --git a/x-pack/plugin/watcher/qa/with-security/build.gradle b/x-pack/plugin/watcher/qa/with-security/build.gradle
index c80b1894d0c73..5a8b03ca68d37 100644
--- a/x-pack/plugin/watcher/qa/with-security/build.gradle
+++ b/x-pack/plugin/watcher/qa/with-security/build.gradle
@@ -7,21 +7,15 @@ dependencies {
   testCompile project(path: ':x-pack:plugin:watcher:qa:rest', configuration: 'testArtifacts')
 }
 
-
-// bring in watcher rest test suite from the rest project
-task copyWatcherRestTests(type: Copy) {
-  into project.sourceSets.test.output.resourcesDir
-  from project(xpackProject('plugin:watcher:qa:rest').path).sourceSets.test.resources.srcDirs
-  include 'rest-api-spec/test/watcher/**'
-}
-
 restResources {
   restApi {
     includeXpack 'watcher', 'security', 'xpack'
   }
+  restTests {
+    includeXpack 'watcher'
+  }
 }
 
-integTest.runner.dependsOn copyWatcherRestTests
 testClusters.integTest {
   testDistribution = 'DEFAULT'
   setting 'xpack.ilm.enabled', 'false'