Refactor watcher tests (#52799)

This PR moves the majority of the Watcher REST tests under
the Watcher x-pack plugin.

Specifically, moves the Watcher tests from:
x-pack/plugin/test
x-pack/qa/smoke-test-watcher
x-pack/qa/smoke-test-watcher-with-security
x-pack/qa/smoke-test-monitoring-with-watcher

to:
x-pack/plugin/watcher/qa/rest (/test and /qa/smoke-test-watcher)
x-pack/plugin/watcher/qa/with-security
x-pack/plugin/watcher/qa/with-monitoring

Additionally, this disables Watcher from the main
x-pack test cluster and consolidates the stop/start logic
for the tests listed.

No changes to the tests (beyond moving them) are included.

3rd party tests and doc tests (which also touch Watcher)
are not included in the changes here.

Author: jakelandis

x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java

@@ -103,7 +103,6 @@ public void reenableWatcher() throws Exception {
         }
     }
 
-    @Override
     protected boolean isWatcherTest() {
         String testName = getTestName();
         return testName != null && (testName.contains("watcher/") || testName.contains("watcher\\"));

x-pack/plugin/build.gradle

@@ -10,7 +10,6 @@ dependencies {
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
-// https://github.com/elastic/x-plugins/issues/724
 configurations {
   testArtifacts.extendsFrom testRuntime
   restXpackSpecs
@@ -88,6 +87,7 @@ testClusters.integTest {
   testDistribution = 'DEFAULT' // this is important since we use the reindex module in ML
   setting 'xpack.ml.enabled', 'true'
   setting 'xpack.security.enabled', 'true'
+  setting 'xpack.watcher.enabled', 'false'
   // Integration tests are supposed to enable/disable exporters before/after each test
   setting 'xpack.monitoring.exporters._local.type', 'local'
   setting 'xpack.monitoring.exporters._local.enabled', 'false'

x-pack/plugin/ml/qa/ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityIT.java

@@ -58,11 +58,6 @@ protected boolean isMonitoringTest() {
         return false;
     }
 
-    @Override
-    protected boolean isWatcherTest() {
-        return false;
-    }
-
     @Override
     protected boolean isMachineLearningTest() {
         return true;

x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java

@@ -6,11 +6,8 @@
 package org.elasticsearch.xpack.test.rest;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import org.apache.http.HttpStatus;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -21,14 +18,12 @@
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.ObjectPath;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
 import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
 import org.elasticsearch.xpack.core.rollup.job.RollupJob;
-import org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField;
 import org.junit.After;
 import org.junit.Before;
 
@@ -50,7 +45,6 @@
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
 
 /** Runs rest tests against external cluster */
 public class XPackRestIT extends ESClientYamlSuiteTestCase {
@@ -77,7 +71,6 @@ protected Settings restClientSettings() {
     @Before
     public void setupForTests() throws Exception {
         waitForTemplates();
-        waitForWatcher();
         enableMonitoring();
     }
 
@@ -103,60 +96,7 @@ private void waitForTemplates() throws Exception {
         }
     }
 
-    private void waitForWatcher() throws Exception {
-        // ensure watcher is started, so that a test can stop watcher and everything still works fine
-        if (isWatcherTest()) {
-            assertBusy(() -> {
-                ClientYamlTestResponse response =
-                    getAdminExecutionContext().callApi("watcher.stats", emptyMap(), emptyList(), emptyMap());
-                String state = (String) response.evaluate("stats.0.watcher_state");
-
-                switch (state) {
-                    case "stopped":
-                        ClientYamlTestResponse startResponse =
-                            getAdminExecutionContext().callApi("watcher.start", emptyMap(), emptyList(), emptyMap());
-                        boolean isAcknowledged = (boolean) startResponse.evaluate("acknowledged");
-                        assertThat(isAcknowledged, is(true));
-                        throw new AssertionError("waiting until stopped state reached started state");
-                    case "stopping":
-                        throw new AssertionError("waiting until stopping state reached stopped state to start again");
-                    case "starting":
-                        throw new AssertionError("waiting until starting state reached started state");
-                    case "started":
-                        // all good here, we are done
-                        break;
-                    default:
-                        throw new AssertionError("unknown state[" + state + "]");
-                }
-            });
 
-            for (String template : WatcherIndexTemplateRegistryField.TEMPLATE_NAMES) {
-                awaitCallApi("indices.exists_template", singletonMap("name", template), emptyList(),
-                    response -> true,
-                    () -> "Exception when waiting for [" + template + "] template to be created");
-            }
-
-            boolean existsWatcherIndex = adminClient()
-                    .performRequest(new Request("HEAD", ".watches"))
-                    .getStatusLine().getStatusCode() == 200;
-            if (existsWatcherIndex == false) {
-                return;
-            }
-            Request searchWatchesRequest = new Request("GET", ".watches/_search");
-            searchWatchesRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");
-            searchWatchesRequest.addParameter("size", "1000");
-            Response response = adminClient().performRequest(searchWatchesRequest);
-            ObjectPath objectPathResponse = ObjectPath.createFromResponse(response);
-            int totalHits = objectPathResponse.evaluate("hits.total");
-            if (totalHits > 0) {
-                List<Map<String, Object>> hits = objectPathResponse.evaluate("hits.hits");
-                for (Map<String, Object> hit : hits) {
-                    String id = (String) hit.get("_id");
-                    adminClient().performRequest(new Request("DELETE", "_watcher/watch/" + id));
-                }
-            }
-        }
-    }
 
     /**
      * Enable monitoring and waits for monitoring documents to be collected and indexed in
@@ -314,11 +254,6 @@ protected boolean isMonitoringTest() {
         return testName != null && (testName.contains("=monitoring/") || testName.contains("=monitoring\\"));
     }
 
-    protected boolean isWatcherTest() {
-        String testName = getTestName();
-        return testName != null && (testName.contains("=watcher/") || testName.contains("=watcher\\"));
-    }
-
     protected boolean isMachineLearningTest() {
         String testName = getTestName();
         return testName != null && (testName.contains("=ml/") || testName.contains("=ml\\"));

x-pack/plugin/watcher/build.gradle

@@ -93,3 +93,12 @@ test {
 // installing them as individual plugins for integ tests doesn't make sense,
 // so we disable integ tests
 integTest.enabled = false
+
+// add all sub-projects of the qa sub-project
+gradle.projectsEvaluated {
+  project.subprojects
+    .find { it.path == project.path + ":qa" }
+    .subprojects
+    .findAll { it.path.startsWith(project.path + ":qa") }
+    .each { check.dependsOn it.check }
+}

x-pack/plugin/watcher/qa/build.gradle

@@ -6,9 +6,22 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
+configurations {
+  testArtifacts.extendsFrom testRuntime
+}
+
+task testJar(type: Jar) {
+  appendix 'test'
+  from sourceSets.test.output
+}
+
+artifacts {
+  testArtifacts testJar
+}
+
 restResources {
   restApi {
-    includeXpack 'watcher'
+    includeXpack 'watcher', 'xpack'
   }
 }
 

x-pack/qa/smoke-test-watcher/build.gradle renamed to x-pack/plugin/watcher/qa/rest/build.gradle

@@ -14,15 +14,11 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
-import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
-import org.junit.After;
-import org.junit.Before;
+import org.elasticsearch.xpack.watcher.WatcherRestTestCase;
 
 import java.io.IOException;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -33,71 +29,11 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SmokeTestWatcherTestSuiteIT extends ESRestTestCase {
+public class SmokeTestWatcherTestSuiteIT extends WatcherRestTestCase {
 
     private static final String TEST_ADMIN_USERNAME = "test_admin";
     private static final String TEST_ADMIN_PASSWORD = "x-pack-test-password";
 
-    @Before
-    public void startWatcher() throws Exception {
-        // delete the watcher history to not clutter with entries from other test
-        assertOK(adminClient().performRequest(new Request("DELETE", "/.watcher-history-*")));
-
-        assertBusy(() -> {
-            Response response = adminClient().performRequest(new Request("GET", "/_watcher/stats"));
-            String state = ObjectPath.createFromResponse(response).evaluate("stats.0.watcher_state");
-
-            switch (state) {
-                case "stopped":
-                    Response startResponse = adminClient().performRequest(new Request("POST", "/_watcher/_start"));
-                    boolean isAcknowledged = ObjectPath.createFromResponse(startResponse).evaluate("acknowledged");
-                    assertThat(isAcknowledged, is(true));
-                    throw new AssertionError("waiting until stopped state reached started state");
-                case "stopping":
-                    throw new AssertionError("waiting until stopping state reached stopped state to start again");
-                case "starting":
-                    throw new AssertionError("waiting until starting state reached started state");
-                case "started":
-                    // all good here, we are done
-                    break;
-                default:
-                    throw new AssertionError("unknown state[" + state + "]");
-            }
-        });
-
-        assertBusy(() -> {
-            for (String template : XPackRestTestConstants.TEMPLATE_NAMES_NO_ILM) {
-                Response templateExistsResponse = adminClient().performRequest(new Request("HEAD", "/_template/" + template));
-                assertThat(templateExistsResponse.getStatusLine().getStatusCode(), is(200));
-            }
-        });
-    }
-
-    @After
-    public void stopWatcher() throws Exception {
-        assertBusy(() -> {
-            Response response = adminClient().performRequest(new Request("GET", "/_watcher/stats"));
-            String state = ObjectPath.createFromResponse(response).evaluate("stats.0.watcher_state");
-
-            switch (state) {
-                case "stopped":
-                    // all good here, we are done
-                    break;
-                case "stopping":
-                    throw new AssertionError("waiting until stopping state reached stopped state");
-                case "starting":
-                    throw new AssertionError("waiting until starting state reached started state to stop");
-                case "started":
-                    Response stopResponse = adminClient().performRequest(new Request("POST", "/_watcher/_stop"));
-                    boolean isAcknowledged = ObjectPath.createFromResponse(stopResponse).evaluate("acknowledged");
-                    assertThat(isAcknowledged, is(true));
-                    throw new AssertionError("waiting until started state reached stopped state");
-                default:
-                    throw new AssertionError("unknown state[" + state + "]");
-            }
-        }, 60, TimeUnit.SECONDS);
-    }
-
     @Override
     protected Settings restClientSettings() {
         String token = basicAuthHeaderValue("watcher_manager", new SecureString("x-pack-test-password".toCharArray()));

x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java renamed to x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.smoketest;
+
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.xpack.watcher.WatcherYamlSuiteTestCase;
+
+/**
+ * Runs the YAML rest tests against an external cluster
+ */
+public class WatcherYamlRestIT extends WatcherYamlSuiteTestCase {
+    public WatcherYamlRestIT(ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+}

x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/smoketest/WatcherYamlRestIT.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.watcher;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.yaml.ObjectPath;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Parent test class for Watcher (not-YAML) based REST tests
+ */
+public abstract class WatcherRestTestCase extends ESRestTestCase {
+
+    @Before
+    public final void startWatcher() throws Exception {
+        assertBusy(() -> {
+            Response response = adminClient().performRequest(new Request("GET", "/_watcher/stats"));
+            String state = ObjectPath.createFromResponse(response).evaluate("stats.0.watcher_state");
+
+            switch (state) {
+                case "stopped":
+                    Response startResponse = adminClient().performRequest(new Request("POST", "/_watcher/_start"));
+                    boolean isAcknowledged = ObjectPath.createFromResponse(startResponse).evaluate("acknowledged");
+                    assertThat(isAcknowledged, is(true));
+                    throw new AssertionError("waiting until stopped state reached started state");
+                case "stopping":
+                    throw new AssertionError("waiting until stopping state reached stopped state to start again");
+                case "starting":
+                    throw new AssertionError("waiting until starting state reached started state");
+                case "started":
+                    // all good here, we are done
+                    break;
+                default:
+                    throw new AssertionError("unknown state[" + state + "]");
+            }
+        });
+    }
+
+    @After
+    public final void stopWatcher() throws Exception {
+        assertBusy(() -> {
+            Response response = adminClient().performRequest(new Request("GET", "/_watcher/stats"));
+            String state = ObjectPath.createFromResponse(response).evaluate("stats.0.watcher_state");
+
+            switch (state) {
+                case "stopped":
+                    // all good here, we are done
+                    break;
+                case "stopping":
+                    throw new AssertionError("waiting until stopping state reached stopped state");
+                case "starting":
+                    throw new AssertionError("waiting until starting state reached started state to stop");
+                case "started":
+                    Response stopResponse = adminClient().performRequest(new Request("POST", "/_watcher/_stop"));
+                    boolean isAcknowledged = ObjectPath.createFromResponse(stopResponse).evaluate("acknowledged");
+                    assertThat(isAcknowledged, is(true));
+                    throw new AssertionError("waiting until started state reached stopped state");
+                default:
+                    throw new AssertionError("unknown state[" + state + "]");
+            }
+        }, 60, TimeUnit.SECONDS);
+
+        Request deleteWatchesIndexRequest = new Request("DELETE", ".watches");
+        deleteWatchesIndexRequest.addParameter("ignore_unavailable", "true");
+        adminClient().performRequest(deleteWatchesIndexRequest);
+
+        Request deleteWatchHistoryRequest = new Request("DELETE", ".watcher-history-*");
+        deleteWatchHistoryRequest.addParameter("ignore_unavailable", "true");
+        adminClient().performRequest(deleteWatchHistoryRequest);
+    }
+}