Merge remote-tracking branch 'elastic/master' into retention-leases-version

* elastic/master:
  SnapshotShardsService Simplifications (elastic#38025)
  Default include_type_name to false in the yml test harness. (elastic#38058)
  Disable bwc preparing to backport of#37977, elastic#37857 and elastic#37872 (elastic#38126)
  Adding ml_settings entry to HLRC and Docs for deprecation_info (elastic#38118)

Author: jasontedor

build.gradle

@@ -160,7 +160,7 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 final boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/37951,https://github.com/elastic/elasticsearch/pull/38180" /* place a PR link here when committing bwc changes */
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/37951" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

client/rest-high-level/src/main/java/org/elasticsearch/client/migration/DeprecationInfoResponse.java

@@ -37,16 +37,19 @@ public class DeprecationInfoResponse {
     private static final ParseField CLUSTER_SETTINGS = new ParseField("cluster_settings");
     private static final ParseField NODE_SETTINGS = new ParseField("node_settings");
     private static final ParseField INDEX_SETTINGS = new ParseField("index_settings");
+    private static final ParseField ML_SETTINGS = new ParseField("ml_settings");
 
     private final List<DeprecationIssue> clusterSettingsIssues;
     private final List<DeprecationIssue> nodeSettingsIssues;
     private final Map<String, List<DeprecationIssue>> indexSettingsIssues;
+    private final List<DeprecationIssue> mlSettingsIssues;
 
     public DeprecationInfoResponse(List<DeprecationIssue> clusterSettingsIssues, List<DeprecationIssue> nodeSettingsIssues,
-                                   Map<String, List<DeprecationIssue>> indexSettingsIssues) {
+                                   Map<String, List<DeprecationIssue>> indexSettingsIssues, List<DeprecationIssue> mlSettingsIssues) {
         this.clusterSettingsIssues = Objects.requireNonNull(clusterSettingsIssues, "cluster settings issues cannot be null");
         this.nodeSettingsIssues = Objects.requireNonNull(nodeSettingsIssues, "node settings issues cannot be null");
         this.indexSettingsIssues = Objects.requireNonNull(indexSettingsIssues, "index settings issues cannot be null");
+        this.mlSettingsIssues = Objects.requireNonNull(mlSettingsIssues, "ml settings issues cannot be null");
     }
 
     public List<DeprecationIssue> getClusterSettingsIssues() {
@@ -61,6 +64,10 @@ public Map<String, List<DeprecationIssue>> getIndexSettingsIssues() {
         return indexSettingsIssues;
     }
 
+    public List<DeprecationIssue> getMlSettingsIssues() {
+        return mlSettingsIssues;
+    }
+
     private static List<DeprecationIssue> parseDeprecationIssues(XContentParser parser) throws IOException {
         List<DeprecationIssue> issues = new ArrayList<>();
         XContentParser.Token token = null;
@@ -76,6 +83,7 @@ public static DeprecationInfoResponse fromXContent(XContentParser parser) throws
         Map<String, List<DeprecationIssue>> indexSettings = new HashMap<>();
         List<DeprecationIssue> clusterSettings = new ArrayList<>();
         List<DeprecationIssue> nodeSettings = new ArrayList<>();
+        List<DeprecationIssue> mlSettings = new ArrayList<>();
         String fieldName = null;
         XContentParser.Token token;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -85,6 +93,8 @@ public static DeprecationInfoResponse fromXContent(XContentParser parser) throws
                 clusterSettings.addAll(parseDeprecationIssues(parser));
             } else if (NODE_SETTINGS.getPreferredName().equals(fieldName)) {
                 nodeSettings.addAll(parseDeprecationIssues(parser));
+            } else if (ML_SETTINGS.getPreferredName().equals(fieldName)) {
+                mlSettings.addAll(parseDeprecationIssues(parser));
             } else if (INDEX_SETTINGS.getPreferredName().equals(fieldName)) {
                 // parse out the key/value pairs
                 while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -96,7 +106,7 @@ public static DeprecationInfoResponse fromXContent(XContentParser parser) throws
                 }
             }
         }
-        return new DeprecationInfoResponse(clusterSettings, nodeSettings, indexSettings);
+        return new DeprecationInfoResponse(clusterSettings, nodeSettings, indexSettings, mlSettings);
     }
 
     @Override
@@ -106,17 +116,19 @@ public boolean equals(Object o) {
         DeprecationInfoResponse that = (DeprecationInfoResponse) o;
         return Objects.equals(clusterSettingsIssues, that.clusterSettingsIssues) &&
             Objects.equals(nodeSettingsIssues, that.nodeSettingsIssues) &&
+            Objects.equals(mlSettingsIssues, that.mlSettingsIssues) &&
             Objects.equals(indexSettingsIssues, that.indexSettingsIssues);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues);
+        return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues);
     }
 
     @Override
     public String toString() {
-        return clusterSettingsIssues.toString() + ":" + nodeSettingsIssues.toString() + ":" + indexSettingsIssues.toString();
+        return clusterSettingsIssues.toString() + ":" + nodeSettingsIssues.toString() + ":" + indexSettingsIssues.toString() +
+            ":" + mlSettingsIssues.toString();
     }
 
     /**

client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java

@@ -85,6 +85,7 @@ public void testGetDeprecationInfo() throws IOException {
         assertThat(response.getClusterSettingsIssues().size(), equalTo(0));
         assertThat(response.getIndexSettingsIssues().size(), equalTo(0));
         assertThat(response.getNodeSettingsIssues().size(), equalTo(0));
+        assertThat(response.getMlSettingsIssues().size(), equalTo(0));
     }
 
     /**

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationClientDocumentationIT.java

@@ -182,6 +182,8 @@ public void testGetDeprecationInfo() throws IOException, InterruptedException {
             deprecationInfoResponse.getNodeSettingsIssues(); // <2>
         Map<String, List<DeprecationInfoResponse.DeprecationIssue>> indexIssues =
             deprecationInfoResponse.getIndexSettingsIssues(); // <3>
+        List<DeprecationInfoResponse.DeprecationIssue> mlIssues =
+            deprecationInfoResponse.getMlSettingsIssues(); // <4>
         // end::get-deprecation-info-response
 
         // tag::get-deprecation-info-execute-listener
@@ -195,6 +197,8 @@ public void onResponse(DeprecationInfoResponse deprecationInfoResponse1) { // <1
                         deprecationInfoResponse.getNodeSettingsIssues();
                     Map<String, List<DeprecationInfoResponse.DeprecationIssue>> indexIssues =
                         deprecationInfoResponse.getIndexSettingsIssues();
+                    List<DeprecationInfoResponse.DeprecationIssue> mlIssues =
+                        deprecationInfoResponse.getMlSettingsIssues();
                 }
 
                 @Override

client/rest-high-level/src/test/java/org/elasticsearch/client/migration/DeprecationInfoResponseTests.java

@@ -65,6 +65,12 @@ private void toXContent(DeprecationInfoResponse response, XContentBuilder builde
                 }
             }
             builder.endObject();
+
+            builder.startArray("ml_settings");
+            for (DeprecationInfoResponse.DeprecationIssue issue : response.getMlSettingsIssues()) {
+                toXContent(issue, builder);
+            }
+            builder.endArray();
         }
         builder.endObject();
     }
@@ -105,12 +111,14 @@ private List<DeprecationInfoResponse.DeprecationIssue> createRandomIssues(boolea
     }
 
     private DeprecationInfoResponse createInstance() {
-        return new DeprecationInfoResponse(createRandomIssues(true), createRandomIssues(true), createIndexSettingsIssues());
+        return new DeprecationInfoResponse(createRandomIssues(true), createRandomIssues(true), createIndexSettingsIssues(),
+            createRandomIssues(true));
     }
 
     private DeprecationInfoResponse copyInstance(DeprecationInfoResponse req) {
         return new DeprecationInfoResponse(new ArrayList<>(req.getClusterSettingsIssues()),
-            new ArrayList<>(req.getNodeSettingsIssues()), new HashMap<>(req.getIndexSettingsIssues()));
+            new ArrayList<>(req.getNodeSettingsIssues()), new HashMap<>(req.getIndexSettingsIssues()),
+            new ArrayList<>(req.getMlSettingsIssues()));
     }
 
     private DeprecationInfoResponse mutateInstance(DeprecationInfoResponse req) {
@@ -128,16 +136,21 @@ public void testFromXContent() throws IOException {
     }
 
     public void testNullFailedIndices() {
-        NullPointerException exception =
-            expectThrows(NullPointerException.class, () -> new DeprecationInfoResponse(null, null, null));
+        NullPointerException exception = expectThrows(NullPointerException.class,
+            () -> new DeprecationInfoResponse(null, null, null, null));
         assertEquals("cluster settings issues cannot be null", exception.getMessage());
 
-        exception = expectThrows(NullPointerException.class, () -> new DeprecationInfoResponse(Collections.emptyList(), null, null));
+        exception = expectThrows(NullPointerException.class,
+            () -> new DeprecationInfoResponse(Collections.emptyList(), null, null, null));
         assertEquals("node settings issues cannot be null", exception.getMessage());
 
         exception = expectThrows(NullPointerException.class,
-            () -> new DeprecationInfoResponse(Collections.emptyList(), Collections.emptyList(), null));
+            () -> new DeprecationInfoResponse(Collections.emptyList(), Collections.emptyList(), null, null));
         assertEquals("index settings issues cannot be null", exception.getMessage());
+
+        exception = expectThrows(NullPointerException.class,
+            () -> new DeprecationInfoResponse(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null));
+        assertEquals("ml settings issues cannot be null", exception.getMessage());
     }
 
     public void testEqualsAndHashCode() {

docs/java-rest/high-level/migration/get-deprecation-info.asciidoc

@@ -33,3 +33,4 @@ include-tagged::{doc-tests-file}[{api}-response]
 <1> a List of Cluster deprecations
 <2> a List of Node deprecations
 <3> a Map of key IndexName, value List of deprecations for the index
+<4> a list of Machine Learning related deprecations

docs/reference/migration/apis/deprecation.asciidoc

@@ -68,7 +68,8 @@ Example response:
         "details" : "This index is named [logs:apache], which contains the illegal character ':'."
       }
     ]
-  }
+  },
+  "ml_settings" : [ ]
 }
 --------------------------------------------------
 // NOTCONSOLE
@@ -109,7 +110,8 @@ key. Similarly, any node-level warnings are found under `node_settings`. Since
 only a select subset of your nodes might incorporate these settings, it is
 important to read the `details` section for more information about which nodes
 are affected. Index warnings are sectioned off per index and can be filtered
-using an index-pattern in the query.
+using an index-pattern in the query. Machine Learning related deprecation
+warnings can be found under the `ml_settings` key.
 
 The following example request shows only index-level deprecations of all
 `logstash-*` indices:

modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yml

@@ -2,7 +2,6 @@
 "Empty Bucket Aggregation":
   - do:
       indices.create:
-        include_type_name: false
         index: empty_bucket_idx
         body:
           settings:

modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml

@@ -3,7 +3,6 @@ setup:
 
   - do:
       indices.create:
-        include_type_name: false
         index: test
         body:
           settings:

modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml

@@ -3,7 +3,6 @@ setup:
 
   - do:
       indices.create:
-        include_type_name: false
         index: test
         body:
           settings: