Merge branch 'master' into feature/searchable-snapshots

# Conflicts:
#	plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
#	plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
#	plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java
#	server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
#	server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
#	x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java
#	x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopyExecutionStateStep.java
#	x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkSetAliasStep.java
#	x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java
#	x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CopyExecutionStateStepTests.java

Author: andreidan

.ci/bwcVersions

@@ -17,6 +17,7 @@ BWC_VERSION:
   - "7.6.0"
   - "7.6.1"
   - "7.6.2"
+  - "7.6.3"
   - "7.7.0"
   - "7.8.0"
   - "8.0.0"

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -21,8 +21,8 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -127,27 +127,27 @@ public void setUp() throws Exception {
             Settings.builder().put("cluster.routing.allocation.awareness.attributes", "tag").build()
         );
 
-        MetaData.Builder mb = MetaData.builder();
+        Metadata.Builder mb = Metadata.builder();
         for (int i = 1; i <= numIndices; i++) {
             mb.put(
-                IndexMetaData.builder("test_" + i)
+                IndexMetadata.builder("test_" + i)
                     .settings(Settings.builder().put("index.version.created", Version.CURRENT))
                     .numberOfShards(numShards)
                     .numberOfReplicas(numReplicas)
             );
         }
-        MetaData metaData = mb.build();
+        Metadata metadata = mb.build();
         RoutingTable.Builder rb = RoutingTable.builder();
         for (int i = 1; i <= numIndices; i++) {
-            rb.addAsNew(metaData.index("test_" + i));
+            rb.addAsNew(metadata.index("test_" + i));
         }
         RoutingTable routingTable = rb.build();
         DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
         for (int i = 1; i <= numNodes; i++) {
             nb.add(Allocators.newNode("node" + i, Collections.singletonMap("tag", "tag_" + (i % numTags))));
         }
         initialClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metaData(metaData)
+            .metadata(metadata)
             .routingTable(routingTable)
             .nodes(nb)
             .build();

build.gradle

@@ -511,8 +511,8 @@ subprojects {
   pluginManager.withPlugin('elasticsearch.testclusters') {
     testClusters.all {
       if (org.elasticsearch.gradle.info.BuildParams.isSnapshotBuild() == false) {
-        systemProperty 'es.itv2_feature_flag_registered', 'true'
-        systemProperty 'es.datastreams_feature_flag_registered', 'true'
+        systemProperty 'es.itv2_feature_enabled', 'true'
+        systemProperty 'es.datastreams_feature_enabled', 'true'
       }
     }
   }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -5,11 +5,16 @@
 
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 public class SystemPropertyCommandLineArgumentProvider implements CommandLineArgumentProvider {
     private final Map<String, Object> systemProperties = new LinkedHashMap<>();
 
+    public void systemProperty(String key, Supplier<String> value) {
+        systemProperties.put(key, value);
+    }
+
     public void systemProperty(String key, Object value) {
         systemProperties.put(key, value);
     }
@@ -18,7 +23,12 @@ public void systemProperty(String key, Object value) {
     public Iterable<String> asArguments() {
         return systemProperties.entrySet()
             .stream()
-            .map(entry -> "-D" + entry.getKey() + "=" + entry.getValue())
+            .map(
+                entry -> "-D"
+                    + entry.getKey()
+                    + "="
+                    + (entry.getValue() instanceof Supplier ? ((Supplier) entry.getValue()).get() : entry.getValue())
+            )
             .collect(Collectors.toList());
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/SystemPropertyCommandLineArgumentProvider.java

@@ -7,7 +7,7 @@
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -16,7 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.gradle.test
+
+package org.elasticsearch.gradle.test;
 
 /**
  * Any object that can produce an accompanying stop task, meant to tear down
@@ -25,6 +26,6 @@
 public interface Fixture {
 
     /** A task which will stop this fixture. This should be used as a finalizedBy for any tasks that use the fixture. */
-    public Object getStopTask()
+    Object getStopTask();
 
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/Fixture.groovy renamed to buildSrc/src/main/java/org/elasticsearch/gradle/test/Fixture.java

@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.test;
+
+import org.elasticsearch.gradle.SystemPropertyCommandLineArgumentProvider;
+import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
+import org.elasticsearch.gradle.testclusters.RestTestRunnerTask;
+import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
+import org.gradle.api.Action;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+
+public class RestIntegTestTask extends DefaultTask {
+
+    protected RestTestRunnerTask runner;
+    private static final String TESTS_REST_CLUSTER = "tests.rest.cluster";
+    private static final String TESTS_CLUSTER = "tests.cluster";
+    private static final String TESTS_CLUSTER_NAME = "tests.clustername";
+
+    public RestIntegTestTask() {
+        Project project = getProject();
+        String name = getName();
+        runner = project.getTasks().create(name + "Runner", RestTestRunnerTask.class);
+        super.dependsOn(runner);
+        @SuppressWarnings("unchecked")
+        NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
+            .getExtensions()
+            .getByName(TestClustersPlugin.EXTENSION_NAME);
+        ElasticsearchCluster cluster = testClusters.create(name);
+        runner.useCluster(cluster);
+        runner.include("**/*IT.class");
+        runner.systemProperty("tests.rest.load_packaged", Boolean.FALSE.toString());
+        if (System.getProperty(TESTS_REST_CLUSTER) == null) {
+            if (System.getProperty(TESTS_CLUSTER) != null || System.getProperty(TESTS_CLUSTER_NAME) != null) {
+                throw new IllegalArgumentException(
+                    String.format("%s, %s, and %s must all be null or non-null", TESTS_REST_CLUSTER, TESTS_CLUSTER, TESTS_CLUSTER_NAME)
+                );
+            }
+            SystemPropertyCommandLineArgumentProvider runnerNonInputProperties = (SystemPropertyCommandLineArgumentProvider) runner
+                .getExtensions()
+                .getByName("nonInputProperties");
+            runnerNonInputProperties.systemProperty(TESTS_REST_CLUSTER, () -> String.join(",", cluster.getAllHttpSocketURI()));
+            runnerNonInputProperties.systemProperty(TESTS_CLUSTER, () -> String.join(",", cluster.getAllTransportPortURI()));
+            runnerNonInputProperties.systemProperty(TESTS_CLUSTER_NAME, cluster::getName);
+        } else {
+            if (System.getProperty(TESTS_CLUSTER) == null || System.getProperty(TESTS_CLUSTER_NAME) == null) {
+                throw new IllegalArgumentException(
+                    String.format("%s, %s, and %s must all be null or non-null", TESTS_REST_CLUSTER, TESTS_CLUSTER, TESTS_CLUSTER_NAME)
+                );
+            }
+        }
+        // this must run after all projects have been configured, so we know any project
+        // references can be accessed as a fully configured
+        project.getGradle().projectsEvaluated(x -> {
+            if (isEnabled() == false) {
+                runner.setEnabled(false);
+            }
+        });
+    }
+
+    @Override
+    public Task dependsOn(Object... dependencies) {
+        runner.dependsOn(dependencies);
+        for (Object dependency : dependencies) {
+            if (dependency instanceof Fixture) {
+                runner.finalizedBy(((Fixture) dependency).getStopTask());
+            }
+        }
+        return this;
+    }
+
+    @Override
+    public void setDependsOn(Iterable<?> dependencies) {
+        runner.setDependsOn(dependencies);
+        for (Object dependency : dependencies) {
+            if (dependency instanceof Fixture) {
+                runner.finalizedBy(((Fixture) dependency).getStopTask());
+            }
+        }
+    }
+
+    public void runner(Action<? super RestTestRunnerTask> configure) {
+        configure.execute(runner);
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/test/RestIntegTestTask.java

@@ -63,18 +63,18 @@ protected SearchRequestExecutor searchRequestExecutor(RestClient client, String
 
     private static final class RestBulkRequestExecutor implements BulkRequestExecutor {
         private final RestClient client;
-        private final String actionMetaData;
+        private final String actionMetadata;
 
         RestBulkRequestExecutor(RestClient client, String index, String type) {
             this.client = client;
-            this.actionMetaData = String.format(Locale.ROOT, "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }%n", index, type);
+            this.actionMetadata = String.format(Locale.ROOT, "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }%n", index, type);
         }
 
         @Override
         public boolean bulkIndex(List<String> bulkData) {
             StringBuilder bulkRequestBody = new StringBuilder();
             for (String bulkItem : bulkData) {
-                bulkRequestBody.append(actionMetaData);
+                bulkRequestBody.append(actionMetadata);
                 bulkRequestBody.append(bulkItem);
                 bulkRequestBody.append("\n");
             }