Merge branch '6.x' into ccr-6.x

* 6.x:
  Force execution of fetch tasks (#31974)
  [TEST] Mute SlackMessageTests.testTemplateRender
  Docs: Explain closing the high level client
  [test] disable java packaging tests for suse
  XContentTests : Insert random fields at random positions (#30867)
  Add Get Snapshots High Level REST API (#31980)
  Fix unreachable error condition in AmazonS3Fixture (#32005)
  [6.x][ML] Ensure immutability of MlMetadata (#31994)
  Add Expected Reciprocal Rank metric (#31891)
  SQL: Add support for single parameter text manipulating functions (#31874)
  muted tests due to #31940
  Work around reported problem in eclipse (#31960)
  Move build integration tests out of :buildSrc project (#31961)
  [Test] Reactive 3rd party tests on CI (#31919)
  Fix assertIngestDocument wrongfully passing (#31913) (#31951)
  SQL: Support for escape sequences (#31884)
  SQL: HAVING clause should accept only aggregates (#31872)
  Docs: fix typo in datehistogram (#31972)
  Switch url repository rest tests to new style requests (#31944)
  Switch reindex tests to new style requests (#31941)
  Switch test framework to new style requests (#31939)
  Docs: Added note about cloud service to installation and getting started
  [DOCS] Removes alternative docker pull example (#31934)
  ingest: date_index_name processor template resolution (#31841)
  Test: fix null failure in watcher test (#31968)
  Watcher: Slack message empty text (#31596)
  Switch low level rest tests to new style Requests (#31938)
  Switch high level rest tests to new style requests (#31937)
  HLREST: Bundle the x-pack protocol project (#31904)
  [ML] Mute test failing due to Java 11 date time format parsing bug (#31899)
  Increase logging level for testStressMaybeFlush
  rolling upgrade should use a replica to prevent relocations while running a scroll
  [test] port archive distribution packaging tests (#31314)
  HLRest: Move xPackInfo() to xPack().info() (#31905)
  Increase logging level for :qa:rolling-upgrade
  Backport: Add template config for Beat state to X-Pack Monitoring (#31809) (#31893)
  Fix building AD URL from domain name (#31849)
  Fix broken NaN check in MovingFunctions#stdDev() (#31888)
  Change trappy float comparison (#31889)
  Add opaque_id to audit logging (#31878)
  add support for is_write_index in put-alias body parsing (#31674)
  Ingest: Enable Templated Fieldnames in Rename (#31690) (#31896)
  Ingest: Add ignore_missing option to RemoveProc (#31693) (#31892)
  [Docs] Fix typo in the Rollup API Quick Reference (#31855)
  Watcher: Add ssl.trust email account setting (#31684)
  [PkiRealm] Invalidate cache on role mappings change (#31510)
  [Security] Check auth scheme case insensitively (#31490)
  HLREST: Add x-pack-info API (#31870)
  Remove link to oss-MSI (#31844)
  Painless: Restructure Definition/Whitelist (#31879)

Author: dnhatn

build.gradle

@@ -429,6 +429,9 @@ allprojects {
     if (isEclipse) {
       // set this so generated dirs will be relative to eclipse build
       project.buildDir = eclipseBuild
+      // Work around https://docs.gradle.org/current/userguide/java_gradle_plugin.html confusing Eclipse by the metadata
+      // it adds to the  classpath
+      project.file("$buildDir/pluginUnderTestMetadata").mkdirs()
     }
     eclipse.classpath.file.whenMerged { classpath ->
       // give each source folder a unique corresponding output folder

buildSrc/build.gradle

@@ -128,6 +128,10 @@ if (project == rootProject) {
     }
     mavenCentral()
   }
+  test {
+    include "**/*Tests.class"
+    exclude "**/*IT.class"
+  }
 }
 
 /*****************************************************************************
@@ -152,6 +156,18 @@ if (project != rootProject) {
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
 
+  // tests can't  be run with randomized test runner
+  // it's fine as we run them as part of :buildSrc
+  test.enabled = false
+  task integTest(type: Test) {
+    exclude "**/*Tests.class"
+    include "**/*IT.class"
+    testClassesDirs = sourceSets.test.output.classesDirs
+    classpath = sourceSets.test.runtimeClasspath
+    inputs.dir(file("src/testKit"))
+  }
+  check.dependsOn(integTest)
+
   // TODO: re-enable once randomizedtesting gradle code is published and removed from here
   licenseHeaders.enabled = false
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -550,9 +550,15 @@ class VagrantTestPlugin implements Plugin<Project> {
                 javaPackagingTest.command = 'ssh'
                 javaPackagingTest.args = ['--command', 'sudo bash "$PACKAGING_TESTS/run-tests.sh"']
             } else {
+                // powershell sessions run over winrm always run as administrator, whether --elevated is passed or not. however
+                // remote sessions have some restrictions on what they can do, such as impersonating another user (or the same user
+                // without administrator elevation), which we need to do for these tests. passing --elevated runs the session
+                // as a scheduled job locally on the vm as a true administrator to get around this limitation
+                //
+                // https://github.com/hashicorp/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/plugins/communicators/winrm/communicator.rb#L195-L225
+                // https://devops-collective-inc.gitbooks.io/secrets-of-powershell-remoting/content/manuscript/accessing-remote-computers.html
                 javaPackagingTest.command = 'winrm'
-                // winrm commands run as administrator
-                javaPackagingTest.args = ['--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
+                javaPackagingTest.args = ['--elevated', '--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
             }
 
             TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path)
@@ -563,7 +569,11 @@ class VagrantTestPlugin implements Plugin<Project> {
                 project.gradle.removeListener(javaPackagingReproListener)
             }
             if (project.extensions.esvagrant.boxes.contains(box)) {
-                packagingTest.dependsOn(javaPackagingTest)
+                // these tests are temporarily disabled for suse boxes while we debug an issue
+                // https://github.com/elastic/elasticsearch/issues/30295
+                if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
+                    packagingTest.dependsOn(javaPackagingTest)
+                }
             }
 
             /*

client/rest-high-level/build.gradle

@@ -1,5 +1,3 @@
-import org.elasticsearch.gradle.precommit.PrecommitTasks
-
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -18,29 +16,86 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
  * specific language governing permissions and limitations
  * under the License.
  */
+
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.gradle.api.XmlProvider
+import org.gradle.api.publish.maven.MavenPublication
+
+buildscript {
+  repositories {
+    maven {
+      url 'https://plugins.gradle.org/m2/'
+    }
+  }
+  dependencies {
+    classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
+  }
+}
+
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
+apply plugin: 'com.github.johnrengelman.shadow'
 
 group = 'org.elasticsearch.client'
 archivesBaseName = 'elasticsearch-rest-high-level-client'
 
 publishing {
-    publications {
-        nebula {
-            artifactId = archivesBaseName
+  publications {
+    nebula(MavenPublication) {
+      artifact shadowJar
+      artifactId = archivesBaseName
+      /*
+       * Configure the pom to include the "shadow" as compile dependencies
+       * because that is how we're using them but remove all other dependencies
+       * because they've been shaded into the jar.
+       */
+      pom.withXml { XmlProvider xml ->
+        Node root = xml.asNode()
+        root.remove(root.dependencies)
+        Node dependenciesNode = root.appendNode('dependencies')
+        project.configurations.shadow.allDependencies.each {
+          if (false == it instanceof SelfResolvingDependency) {
+            Node dependencyNode = dependenciesNode.appendNode('dependency')
+            dependencyNode.appendNode('groupId', it.group)
+            dependencyNode.appendNode('artifactId', it.name)
+            dependencyNode.appendNode('version', it.version)
+            dependencyNode.appendNode('scope', 'compile')
+          }
         }
+      }
     }
+  }
+}
+
+/*
+ * We need somewhere to configure dependencies that we don't wish to shade
+ * into the high level REST client. The shadow plugin creates a "shadow"
+ * configuration which is *almost* exactly that. It is never bundled into
+ * the shaded jar but is used for main source compilation. Unfortunately,
+ * by default it is not used for *test* source compilation and isn't used
+ * in tests at all. This change makes it available for test compilation.
+ * A change below makes it available for testing.
+ */
+sourceSets {
+  test {
+    compileClasspath += configurations.shadow
+  }
 }
 
 dependencies {
-  compile "org.elasticsearch:elasticsearch:${version}"
-  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
-  compile "org.elasticsearch.plugin:parent-join-client:${version}"
-  compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
-  compile "org.elasticsearch.plugin:rank-eval-client:${version}"
-  compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
+  /*
+   * Everything in the "shadow" configuration is *not* copied into the
+   * shadowJar.
+   */
+  shadow "org.elasticsearch:elasticsearch:${version}"
+  shadow "org.elasticsearch.client:elasticsearch-rest-client:${version}"
+  shadow "org.elasticsearch.plugin:parent-join-client:${version}"
+  shadow "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
+  shadow "org.elasticsearch.plugin:rank-eval-client:${version}"
+  shadow "org.elasticsearch.plugin:lang-mustache-client:${version}"
+  compile project(':x-pack:protocol')
 
   testCompile "org.elasticsearch.client:test:${version}"
   testCompile "org.elasticsearch.test:framework:${version}"
@@ -63,3 +118,48 @@ forbiddenApisMain {
   signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
   signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
 }
+
+shadowJar {
+  classifier = null
+  mergeServiceFiles()
+}
+
+// We don't need normal jar, we use shadow jar instead
+jar.enabled = false
+assemble.dependsOn shadowJar
+
+javadoc {
+  /*
+   * Bundle all of the javadoc from all of the shaded projects into this one
+   * so we don't *have* to publish javadoc for all of the "client" jars.
+   */
+  configurations.compile.dependencies.all { Dependency dep ->
+    Project p = dependencyToProject(dep)
+    if (p != null) {
+      evaluationDependsOn(p.path)
+      source += p.sourceSets.main.allJava
+    }
+  }
+}
+
+/*
+ * Use the jar for testing so we have tests of the bundled jar.
+ * Use the "shadow" configuration for testing because we need things
+ * in it.
+ */
+test {
+  classpath -= compileJava.outputs.files
+  classpath -= configurations.compile
+  classpath -= configurations.runtime
+  classpath += configurations.shadow
+  classpath += shadowJar.outputs.files
+  dependsOn shadowJar
+}
+integTestRunner {
+  classpath -= compileJava.outputs.files
+  classpath -= configurations.compile
+  classpath -= configurations.runtime
+  classpath += configurations.shadow
+  classpath += shadowJar.outputs.files
+  dependsOn shadowJar
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -39,6 +39,7 @@
 import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
@@ -103,6 +104,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.rankeval.RankEvalRequest;
+import org.elasticsearch.protocol.xpack.XPackInfoRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
 import org.elasticsearch.script.mustache.SearchTemplateRequest;
@@ -114,8 +116,10 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.Charset;
+import java.util.EnumSet;
 import java.util.Locale;
 import java.util.StringJoiner;
+import java.util.stream.Collectors;
 
 final class RequestConverters {
     static final XContentType REQUEST_BODY_CONTENT_TYPE = XContentType.JSON;
@@ -950,6 +954,26 @@ static Request createSnapshot(CreateSnapshotRequest createSnapshotRequest) throw
         return request;
     }
 
+    static Request getSnapshots(GetSnapshotsRequest getSnapshotsRequest) {
+        EndpointBuilder endpointBuilder = new EndpointBuilder().addPathPartAsIs("_snapshot")
+            .addPathPart(getSnapshotsRequest.repository());
+        String endpoint;
+        if (getSnapshotsRequest.snapshots().length == 0) {
+            endpoint = endpointBuilder.addPathPart("_all").build();
+        } else {
+            endpoint = endpointBuilder.addCommaSeparatedPathParts(getSnapshotsRequest.snapshots()).build();
+        }
+
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+
+        Params parameters = new Params(request);
+        parameters.withMasterTimeout(getSnapshotsRequest.masterNodeTimeout());
+        parameters.putParam("ignore_unavailable", Boolean.toString(getSnapshotsRequest.ignoreUnavailable()));
+        parameters.putParam("verbose", Boolean.toString(getSnapshotsRequest.verbose()));
+
+        return request;
+    }
+
     static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) {
         String endpoint = new EndpointBuilder().addPathPartAsIs("_snapshot")
             .addPathPart(deleteSnapshotRequest.repository())
@@ -1055,6 +1079,19 @@ static Request deleteScript(DeleteStoredScriptRequest deleteStoredScriptRequest)
         return request;
     }
 
+    static Request xPackInfo(XPackInfoRequest infoRequest) {
+        Request request = new Request(HttpGet.METHOD_NAME, "/_xpack");
+        if (false == infoRequest.isVerbose()) {
+            request.addParameter("human", "false");
+        }
+        if (false == infoRequest.getCategories().equals(EnumSet.allOf(XPackInfoRequest.Category.class))) {
+            request.addParameter("categories", infoRequest.getCategories().stream()
+                    .map(c -> c.toString().toLowerCase(Locale.ROOT))
+                    .collect(Collectors.joining(",")));
+        }
+        return request;
+    }
+
     private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
         return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -202,6 +202,7 @@ public class RestHighLevelClient implements Closeable {
     private final IngestClient ingestClient = new IngestClient(this);
     private final SnapshotClient snapshotClient = new SnapshotClient(this);
     private final TasksClient tasksClient = new TasksClient(this);
+    private final XPackClient xPackClient = new XPackClient(this);
 
     /**
      * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the
@@ -292,6 +293,19 @@ public final TasksClient tasks() {
         return tasksClient;
     }
 
+    /**
+     * A wrapper for the {@link RestHighLevelClient} that provides methods for
+     * accessing the Elastic Licensed X-Pack APIs that are shipped with the
+     * default distribution of Elasticsearch. All of these APIs will 404 if run
+     * against the OSS distribution of Elasticsearch.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/xpack-api.html">
+     * X-Pack APIs on elastic.co</a> for more information.
+     */
+    public final XPackClient xpack() {
+        return xPackClient;
+    }
+
     /**
      * Executes a bulk request using the Bulk API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html">Bulk API on elastic.co</a>
@@ -979,7 +993,6 @@ public final void rankEvalAsync(RankEvalRequest rankEvalRequest, RequestOptions
                 emptySet());
     }
 
-        
     /**
      * Executes a request using the Multi Search Template API.
      *
@@ -989,9 +1002,9 @@ public final void rankEvalAsync(RankEvalRequest rankEvalRequest, RequestOptions
     public final MultiSearchTemplateResponse multiSearchTemplate(MultiSearchTemplateRequest multiSearchTemplateRequest,
             RequestOptions options) throws IOException {
         return performRequestAndParseEntity(multiSearchTemplateRequest, RequestConverters::multiSearchTemplate,
-                options, MultiSearchTemplateResponse::fromXContext, emptySet());        
-    }   
-    
+                options, MultiSearchTemplateResponse::fromXContext, emptySet());
+    }
+
     /**
      * Asynchronously executes a request using the Multi Search Template API
      *
@@ -1003,7 +1016,7 @@ public final void multiSearchTemplateAsync(MultiSearchTemplateRequest multiSearc
                                           ActionListener<MultiSearchTemplateResponse> listener) {
         performRequestAsyncAndParseEntity(multiSearchTemplateRequest, RequestConverters::multiSearchTemplate,
             options, MultiSearchTemplateResponse::fromXContext, listener, emptySet());
-    }    
+    }
 
     /**
      * Asynchronously executes a request using the Ranking Evaluation API.

client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java

@@ -32,6 +32,8 @@
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 
 import java.io.IOException;
 
@@ -190,6 +192,35 @@ public void createSnapshotAsync(CreateSnapshotRequest createSnapshotRequest, Req
             CreateSnapshotResponse::fromXContent, listener, emptySet());
     }
 
+    /**
+     * Get snapshots.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
+     * API on elastic.co</a>
+     *
+     * @param getSnapshotsRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetSnapshotsResponse get(GetSnapshotsRequest getSnapshotsRequest, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(getSnapshotsRequest, RequestConverters::getSnapshots, options,
+            GetSnapshotsResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously get snapshots.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
+     * API on elastic.co</a>
+     *
+     * @param getSnapshotsRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void getAsync(GetSnapshotsRequest getSnapshotsRequest, RequestOptions options, ActionListener<GetSnapshotsResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(getSnapshotsRequest, RequestConverters::getSnapshots, options,
+            GetSnapshotsResponse::fromXContent, listener, emptySet());
+    }
+
     /**
      * Deletes a snapshot.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore