Check for .watches that wasn't upgraded properly (#39609)

If the `.watches` index was created in 5.6 and is still a concrete
index, rather than an alias for `.watches-6`, that means that it was not
properly upgraded with the Migration Upgrade API before upgrading to
Elasticsearch 6.x. In this case, calling the Migration Upgrade API will
resolve the problem with the `.watches` index.

This isn't going to be a common case, as Watcher will not run properly
in this situation, but we should handle it and notify the user of the
correct action, just in case.

Author: gwbrown

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java

@@ -112,6 +112,12 @@ static DeprecationIssue oldIndicesCheck(IndexMetaData indexMetaData) {
                     "The .tasks index was created before version 6.0 and cannot be opened in 7.0. " +
                         "You must delete this index and allow it to be re-created by Elasticsearch. If you wish to preserve task history, "+
                         "reindex this index to a new index before deleting it.");
+            } else if (".watches".equals(indexMetaData.getIndex().getName())) {
+                return new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
+                    ".watches was not properly upgraded before upgrading to Elasticsearch 6",
+                    "https://www.elastic.co/guide/en/elasticsearch/reference/current/migration-api-upgrade.html",
+                    "The .watches index was created before version 6.0, and was not properly upgraded in 5.6. " +
+                        "Please upgrade this index using the Migration Upgrade API.");
             }
             if ((mappingCount == 2 && !hasDefaultMapping)
                 || mappingCount > 2) {

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java

@@ -63,6 +63,23 @@ public void testOldTasksIndexCheck() {
         assertEquals(singletonList(expected), issues);
     }
 
+    public void testUnupgradedWatcherIndexCheck() {
+        Version createdWith = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0,
+            VersionUtils.getPreviousVersion(Version.V_6_0_0));
+        IndexMetaData indexMetaData = IndexMetaData.builder(".watches")
+            .settings(settings(createdWith))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .build();
+        DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
+            ".watches was not properly upgraded before upgrading to Elasticsearch 6",
+            "https://www.elastic.co/guide/en/elasticsearch/reference/current/migration-api-upgrade.html",
+            "The .watches index was created before version 6.0, and was not properly upgraded in 5.6. " +
+                "Please upgrade this index using the Migration Upgrade API.");
+        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetaData));
+        assertEquals(singletonList(expected), issues);
+    }
+
     public void testMultipleTypesCheckWithDefaultMapping() throws IOException {
         String mappingName1 = randomAlphaOfLengthBetween(2, 5);
         String mappingJson1 = "{\n" +