Merge branch 'master' into ccr

* master:
  [TEST] Mute SlackMessageTests.testTemplateRender
  Docs: Explain closing the high level client
  [ML] Re-enable memory limit integration tests (#31328)
  [test] disable packaging tests for suse boxes
  Add nio transport to security plugin (#31942)
  XContentTests : Insert random fields at random positions (#30867)
  Force execution of fetch tasks (#31974)
  Fix unreachable error condition in AmazonS3Fixture (#32005)
  Tests: Fix SearchFieldsIT.testDocValueFields (#31995)
  Add Expected Reciprocal Rank metric (#31891)
  [ML] Get ForecastRequestStats doc in RestoreModelSnapshotIT (#31973)
  SQL: Add support for single parameter text manipulating functions (#31874)
  [ML] Ensure immutability of MlMetadata (#31957)
  Tests: Mute SearchFieldsIT.testDocValueFields()
  muted tests due to #31940
  Work around reported problem in eclipse (#31960)
  Move build integration tests out of :buildSrc project (#31961)
  Tests: Remove use of joda time in some tests (#31922)
  [Test] Reactive 3rd party tests on CI (#31919)
  SQL: Support for escape sequences (#31884)
  SQL: HAVING clause should accept only aggregates (#31872)
  Docs: fix typo in datehistogram (#31972)
  Switch url repository rest tests to new style requests (#31944)
  Switch reindex tests to new style requests (#31941)
  Docs: Added note about cloud service to installation and getting started
  [DOCS] Removes alternative docker pull example (#31934)
  Add Snapshots Status API to High Level Rest Client (#31515)
  ingest: date_index_name processor template resolution (#31841)
  Test: fix null failure in watcher test (#31968)
  Switch test framework to new style requests (#31939)
  Switch low level rest tests to new style Requests (#31938)
  Switch high level rest tests to new style requests (#31937)
  [ML] Mute test failing due to Java 11 date time format parsing bug (#31899)
  [TEST] Mute SlackMessageTests.testTemplateRender
  Fix assertIngestDocument wrongfully passing (#31913)
  Remove unused reference to filePermissionsCache (#31923)
  rolling upgrade should use a replica to prevent relocations while running a scroll
  HLREST: Bundle the x-pack protocol project (#31904)
  Increase logging level for testStressMaybeFlush
  Added lenient flag for synonym token filter (#31484)
  [X-Pack] Beats centralized management: security role + licensing (#30520)
  HLRest: Move xPackInfo() to xPack().info() (#31905)
  Docs: add security delete role to api call table (#31907)
  [test] port archive distribution packaging tests (#31314)
  Watcher: Slack message empty text (#31596)
  [ML] Mute failing DetectionRulesIT.testCondition() test
  Fix broken NaN check in MovingFunctions#stdDev() (#31888)
  Date: Add DateFormatters class that uses java.time (#31856)
  [ML] Switch native QA tests to a 3 node cluster (#31757)
  Change trappy float comparison (#31889)
  Fix building AD URL from domain name (#31849)
  Add opaque_id to audit logging (#31878)
  re-enable backcompat tests
  add support for is_write_index in put-alias body parsing (#31674)
  Improve release notes script (#31833)
  [DOCS] Fix broken link in painless example
  Handle missing values in painless (#30975)
  Remove the ability to index or query context suggestions without context (#31007)
  Ingest: Enable Templated Fieldnames in Rename (#31690)
  [Docs] Fix typo in the Rollup API Quick Reference (#31855)
  Ingest: Add ignore_missing option to RemoveProc (#31693)
  Add template config for Beat state to X-Pack Monitoring (#31809)
  Watcher: Add ssl.trust email account setting (#31684)
  Remove link to oss-MSI (#31844)
  Painless: Restructure Definition/Whitelist (#31879)
  HLREST: Add x-pack-info API (#31870)

Author: dnhatn

build.gradle

@@ -435,6 +435,9 @@ allprojects {
     if (isEclipse) {
       // set this so generated dirs will be relative to eclipse build
       project.buildDir = eclipseBuild
+      // Work around https://docs.gradle.org/current/userguide/java_gradle_plugin.html confusing Eclipse by the metadata
+      // it adds to the  classpath
+      project.file("$buildDir/pluginUnderTestMetadata").mkdirs()
     }
     eclipse.classpath.file.whenMerged { classpath ->
       // give each source folder a unique corresponding output folder

buildSrc/build.gradle

@@ -128,6 +128,10 @@ if (project == rootProject) {
     }
     mavenCentral()
   }
+  test {
+    include "**/*Tests.class"
+    exclude "**/*IT.class"
+  }
 }
 
 /*****************************************************************************
@@ -152,6 +156,18 @@ if (project != rootProject) {
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
 
+  // tests can't  be run with randomized test runner
+  // it's fine as we run them as part of :buildSrc
+  test.enabled = false
+  task integTest(type: Test) {
+    exclude "**/*Tests.class"
+    include "**/*IT.class"
+    testClassesDirs = sourceSets.test.output.classesDirs
+    classpath = sourceSets.test.runtimeClasspath
+    inputs.dir(file("src/testKit"))
+  }
+  check.dependsOn(integTest)
+
   // TODO: re-enable once randomizedtesting gradle code is published and removed from here
   licenseHeaders.enabled = false
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -691,6 +691,7 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.task', path
             systemProperty 'tests.security.manager', 'true'
             systemProperty 'jna.nosys', 'true'
+            systemProperty 'es.scripting.exception_for_missing_value', 'true'
             // TODO: remove setting logging level via system property
             systemProperty 'tests.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -526,7 +526,11 @@ class VagrantTestPlugin implements Plugin<Project> {
                     project.gradle.removeListener(batsPackagingReproListener)
                 }
                 if (project.extensions.esvagrant.boxes.contains(box)) {
-                    packagingTest.dependsOn(batsPackagingTest)
+                    // these tests are temporarily disabled for suse boxes while we debug an issue
+                    // https://github.com/elastic/elasticsearch/issues/30295
+                    if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
+                        packagingTest.dependsOn(batsPackagingTest)
+                    }
                 }
             }
 
@@ -546,9 +550,15 @@ class VagrantTestPlugin implements Plugin<Project> {
                 javaPackagingTest.command = 'ssh'
                 javaPackagingTest.args = ['--command', 'sudo bash "$PACKAGING_TESTS/run-tests.sh"']
             } else {
+                // powershell sessions run over winrm always run as administrator, whether --elevated is passed or not. however
+                // remote sessions have some restrictions on what they can do, such as impersonating another user (or the same user
+                // without administrator elevation), which we need to do for these tests. passing --elevated runs the session
+                // as a scheduled job locally on the vm as a true administrator to get around this limitation
+                //
+                // https://github.com/hashicorp/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/plugins/communicators/winrm/communicator.rb#L195-L225
+                // https://devops-collective-inc.gitbooks.io/secrets-of-powershell-remoting/content/manuscript/accessing-remote-computers.html
                 javaPackagingTest.command = 'winrm'
-                // winrm commands run as administrator
-                javaPackagingTest.args = ['--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
+                javaPackagingTest.args = ['--elevated', '--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
             }
 
             TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path)
@@ -559,7 +569,11 @@ class VagrantTestPlugin implements Plugin<Project> {
                 project.gradle.removeListener(javaPackagingReproListener)
             }
             if (project.extensions.esvagrant.boxes.contains(box)) {
-                packagingTest.dependsOn(javaPackagingTest)
+                // these tests are temporarily disabled for suse boxes while we debug an issue
+                // https://github.com/elastic/elasticsearch/issues/30295
+                if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
+                    packagingTest.dependsOn(javaPackagingTest)
+                }
             }
 
             /*

client/rest-high-level/build.gradle

@@ -1,5 +1,3 @@
-import org.elasticsearch.gradle.precommit.PrecommitTasks
-
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -18,29 +16,86 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
  * specific language governing permissions and limitations
  * under the License.
  */
+
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.gradle.api.XmlProvider
+import org.gradle.api.publish.maven.MavenPublication
+
+buildscript {
+  repositories {
+    maven {
+      url 'https://plugins.gradle.org/m2/'
+    }
+  }
+  dependencies {
+    classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
+  }
+}
+
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
+apply plugin: 'com.github.johnrengelman.shadow'
 
 group = 'org.elasticsearch.client'
 archivesBaseName = 'elasticsearch-rest-high-level-client'
 
 publishing {
-    publications {
-        nebula {
-            artifactId = archivesBaseName
+  publications {
+    nebula(MavenPublication) {
+      artifact shadowJar
+      artifactId = archivesBaseName
+      /*
+       * Configure the pom to include the "shadow" as compile dependencies
+       * because that is how we're using them but remove all other dependencies
+       * because they've been shaded into the jar.
+       */
+      pom.withXml { XmlProvider xml ->
+        Node root = xml.asNode()
+        root.remove(root.dependencies)
+        Node dependenciesNode = root.appendNode('dependencies')
+        project.configurations.shadow.allDependencies.each {
+          if (false == it instanceof SelfResolvingDependency) {
+            Node dependencyNode = dependenciesNode.appendNode('dependency')
+            dependencyNode.appendNode('groupId', it.group)
+            dependencyNode.appendNode('artifactId', it.name)
+            dependencyNode.appendNode('version', it.version)
+            dependencyNode.appendNode('scope', 'compile')
+          }
         }
+      }
     }
+  }
+}
+
+/*
+ * We need somewhere to configure dependencies that we don't wish to shade
+ * into the high level REST client. The shadow plugin creates a "shadow"
+ * configuration which is *almost* exactly that. It is never bundled into
+ * the shaded jar but is used for main source compilation. Unfortunately,
+ * by default it is not used for *test* source compilation and isn't used
+ * in tests at all. This change makes it available for test compilation.
+ * A change below makes it available for testing.
+ */
+sourceSets {
+  test {
+    compileClasspath += configurations.shadow
+  }
 }
 
 dependencies {
-  compile "org.elasticsearch:elasticsearch:${version}"
-  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
-  compile "org.elasticsearch.plugin:parent-join-client:${version}"
-  compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
-  compile "org.elasticsearch.plugin:rank-eval-client:${version}"
-  compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
+  /*
+   * Everything in the "shadow" configuration is *not* copied into the
+   * shadowJar.
+   */
+  shadow "org.elasticsearch:elasticsearch:${version}"
+  shadow "org.elasticsearch.client:elasticsearch-rest-client:${version}"
+  shadow "org.elasticsearch.plugin:parent-join-client:${version}"
+  shadow "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
+  shadow "org.elasticsearch.plugin:rank-eval-client:${version}"
+  shadow "org.elasticsearch.plugin:lang-mustache-client:${version}"
+  compile project(':x-pack:protocol')
 
   testCompile "org.elasticsearch.client:test:${version}"
   testCompile "org.elasticsearch.test:framework:${version}"
@@ -63,3 +118,48 @@ forbiddenApisMain {
   signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
   signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
 }
+
+shadowJar {
+  classifier = null
+  mergeServiceFiles()
+}
+
+// We don't need normal jar, we use shadow jar instead
+jar.enabled = false
+assemble.dependsOn shadowJar
+
+javadoc {
+  /*
+   * Bundle all of the javadoc from all of the shaded projects into this one
+   * so we don't *have* to publish javadoc for all of the "client" jars.
+   */
+  configurations.compile.dependencies.all { Dependency dep ->
+    Project p = dependencyToProject(dep)
+    if (p != null) {
+      evaluationDependsOn(p.path)
+      source += p.sourceSets.main.allJava
+    }
+  }
+}
+
+/*
+ * Use the jar for testing so we have tests of the bundled jar.
+ * Use the "shadow" configuration for testing because we need things
+ * in it.
+ */
+test {
+  classpath -= compileJava.outputs.files
+  classpath -= configurations.compile
+  classpath -= configurations.runtime
+  classpath += configurations.shadow
+  classpath += shadowJar.outputs.files
+  dependsOn shadowJar
+}
+integTestRunner {
+  classpath -= compileJava.outputs.files
+  classpath -= configurations.compile
+  classpath -= configurations.runtime
+  classpath += configurations.shadow
+  classpath += shadowJar.outputs.files
+  dependsOn shadowJar
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -43,6 +43,7 @@
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
@@ -104,6 +105,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.rankeval.RankEvalRequest;
+import org.elasticsearch.protocol.xpack.XPackInfoRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
 import org.elasticsearch.script.mustache.SearchTemplateRequest;
@@ -115,8 +117,10 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.Charset;
+import java.util.EnumSet;
 import java.util.Locale;
 import java.util.StringJoiner;
+import java.util.stream.Collectors;
 
 final class RequestConverters {
     static final XContentType REQUEST_BODY_CONTENT_TYPE = XContentType.JSON;
@@ -960,6 +964,20 @@ static Request getSnapshots(GetSnapshotsRequest getSnapshotsRequest) {
         return request;
     }
 
+    static Request snapshotsStatus(SnapshotsStatusRequest snapshotsStatusRequest) {
+        String endpoint = new EndpointBuilder().addPathPartAsIs("_snapshot")
+            .addPathPart(snapshotsStatusRequest.repository())
+            .addCommaSeparatedPathParts(snapshotsStatusRequest.snapshots())
+            .addPathPartAsIs("_status")
+            .build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+
+        Params parameters = new Params(request);
+        parameters.withMasterTimeout(snapshotsStatusRequest.masterNodeTimeout());
+        parameters.withIgnoreUnavailable(snapshotsStatusRequest.ignoreUnavailable());
+        return request;
+    }
+
     static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) {
         String endpoint = new EndpointBuilder().addPathPartAsIs("_snapshot")
             .addPathPart(deleteSnapshotRequest.repository())
@@ -1065,6 +1083,19 @@ static Request deleteScript(DeleteStoredScriptRequest deleteStoredScriptRequest)
         return request;
     }
 
+    static Request xPackInfo(XPackInfoRequest infoRequest) {
+        Request request = new Request(HttpGet.METHOD_NAME, "/_xpack");
+        if (false == infoRequest.isVerbose()) {
+            request.addParameter("human", "false");
+        }
+        if (false == infoRequest.getCategories().equals(EnumSet.allOf(XPackInfoRequest.Category.class))) {
+            request.addParameter("categories", infoRequest.getCategories().stream()
+                    .map(c -> c.toString().toLowerCase(Locale.ROOT))
+                    .collect(Collectors.joining(",")));
+        }
+        return request;
+    }
+
     private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
         return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));
@@ -1246,7 +1277,7 @@ Params withWaitForActiveShards(ActiveShardCount activeShardCount, ActiveShardCou
         }
 
         Params withIndicesOptions(IndicesOptions indicesOptions) {
-            putParam("ignore_unavailable", Boolean.toString(indicesOptions.ignoreUnavailable()));
+            withIgnoreUnavailable(indicesOptions.ignoreUnavailable());
             putParam("allow_no_indices", Boolean.toString(indicesOptions.allowNoIndices()));
             String expandWildcards;
             if (indicesOptions.expandWildcardsOpen() == false && indicesOptions.expandWildcardsClosed() == false) {
@@ -1265,6 +1296,12 @@ Params withIndicesOptions(IndicesOptions indicesOptions) {
             return this;
         }
 
+        Params withIgnoreUnavailable(boolean ignoreUnavailable) {
+            // Always explicitly place the ignore_unavailable value.
+            putParam("ignore_unavailable", Boolean.toString(ignoreUnavailable));
+            return this;
+        }
+
         Params withHuman(boolean human) {
             if (human) {
                 putParam("human", Boolean.toString(human));

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -202,6 +202,7 @@ public class RestHighLevelClient implements Closeable {
     private final IngestClient ingestClient = new IngestClient(this);
     private final SnapshotClient snapshotClient = new SnapshotClient(this);
     private final TasksClient tasksClient = new TasksClient(this);
+    private final XPackClient xPackClient = new XPackClient(this);
 
     /**
      * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the
@@ -292,6 +293,19 @@ public final TasksClient tasks() {
         return tasksClient;
     }
 
+    /**
+     * A wrapper for the {@link RestHighLevelClient} that provides methods for
+     * accessing the Elastic Licensed X-Pack APIs that are shipped with the
+     * default distribution of Elasticsearch. All of these APIs will 404 if run
+     * against the OSS distribution of Elasticsearch.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/xpack-api.html">
+     * X-Pack APIs on elastic.co</a> for more information.
+     */
+    public final XPackClient xpack() {
+        return xPackClient;
+    }
+
     /**
      * Executes a bulk request using the Bulk API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html">Bulk API on elastic.co</a>
@@ -668,7 +682,7 @@ public final RankEvalResponse rankEval(RankEvalRequest rankEvalRequest, RequestO
                 emptySet());
     }
 
-        
+
     /**
      * Executes a request using the Multi Search Template API.
      *
@@ -678,9 +692,9 @@ public final RankEvalResponse rankEval(RankEvalRequest rankEvalRequest, RequestO
     public final MultiSearchTemplateResponse multiSearchTemplate(MultiSearchTemplateRequest multiSearchTemplateRequest,
             RequestOptions options) throws IOException {
         return performRequestAndParseEntity(multiSearchTemplateRequest, RequestConverters::multiSearchTemplate,
-                options, MultiSearchTemplateResponse::fromXContext, emptySet());        
-    }   
-    
+                options, MultiSearchTemplateResponse::fromXContext, emptySet());
+    }
+
     /**
      * Asynchronously executes a request using the Multi Search Template API
      *
@@ -692,7 +706,7 @@ public final void multiSearchTemplateAsync(MultiSearchTemplateRequest multiSearc
                                           ActionListener<MultiSearchTemplateResponse> listener) {
         performRequestAsyncAndParseEntity(multiSearchTemplateRequest, RequestConverters::multiSearchTemplate,
             options, MultiSearchTemplateResponse::fromXContext, listener, emptySet());
-    }    
+    }
 
     /**
      * Asynchronously executes a request using the Ranking Evaluation API.