Merge remote-tracking branch 'elastic/master' into opt-out-query-cache-unsafe-publication

* elastic/master:
  Update contributing docs to reflect JDK 11 (elastic#40955)
  Docs: Simplifying setup by using module configuration variant syntax (elastic#40879)
  Unmute CreateIndexIT testCreateAndDeleteIndexConcurrently CI failures (elastic#40960)
  Revert "Short-circuit rebalancing when disabled (elastic#40942)"
  Mute EnableAllocationShortCircuitTests
  SQL: Refactor args verification of In & conditionals (elastic#40916)
  Avoid sharing source directories as it breaks intellij (elastic#40877)
  Short-circuit rebalancing when disabled (elastic#40942)
  SQL: Prefer resultSets over exceptions in metadata (elastic#40641)
  Mute ClusterPrivilegeTests.testThatSnapshotAndRestore
  Fix Race in AsyncTwoPhaseIndexerTests.testStateMachine (elastic#40947)
  Relax Overly Strict Assertion in TransportShardBulkAction (elastic#40940)

Author: jasontedor

CONTRIBUTING.md

@@ -95,10 +95,10 @@ Contributing to the Elasticsearch codebase
 JDK 12 is required to build Elasticsearch. You must have a JDK 12 installation
 with the environment variable `JAVA_HOME` referencing the path to Java home for
 your JDK 12 installation. By default, tests use the same runtime as `JAVA_HOME`.
-However, since Elasticsearch supports JDK 8, the build supports compiling with
-JDK 12 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
-pointing to the Java home of a JDK 8 installation. Note that this mechanism can
-be used to test against other JDKs as well, this is not only limited to JDK 8.
+However, since Elasticsearch supports JDK 11, the build supports compiling with
+JDK 12 and testing on a JDK 11 runtime; to do this, set `RUNTIME_JAVA_HOME`
+pointing to the Java home of a JDK 11 installation. Note that this mechanism can
+be used to test against other JDKs as well, this is not only limited to JDK 11.
 
 > Note: It is also required to have `JAVA8_HOME`, `JAVA9_HOME`, `JAVA10_HOME`
 and `JAVA11_HOME` available so that the tests can pass.

build.gradle

@@ -338,14 +338,6 @@ gradle.projectsEvaluated {
       integTest.mustRunAfter test
     }
     configurations.all { Configuration configuration ->
-      /*
-       * The featureAwarePlugin configuration has a dependency on x-pack:plugin:core and x-pack:plugin:core has a dependency on the
-       * featureAwarePlugin configuration. The below task ordering logic would force :x-pack:plugin:core:test
-       * :x-pack:test:feature-aware:test to depend on each other circularly. We break that cycle here.
-       */
-      if (configuration.name == "featureAwarePlugin") {
-        return
-      }
       dependencies.all { Dependency dep ->
         Project upstreamProject = dependencyToProject(dep)
         if (upstreamProject != null) {
@@ -357,7 +349,7 @@ gradle.projectsEvaluated {
             Task task = project.tasks.findByName(taskName)
             Task upstreamTask = upstreamProject.tasks.findByName(taskName)
             if (task != null && upstreamTask != null) {
-              task.mustRunAfter(upstreamTask)
+              task.shouldRunAfter(upstreamTask)
             }
           }
         }
@@ -382,21 +374,6 @@ allprojects {
       // also ignore other possible build dirs
       excludeDirs += file('build')
       excludeDirs += file('build-eclipse')
-
-      iml {
-        // fix so that Gradle idea plugin properly generates support for resource folders
-        // see also https://issues.gradle.org/browse/GRADLE-2975
-        withXml {
-          it.asNode().component.content.sourceFolder.findAll { it.@url == 'file://$MODULE_DIR$/src/main/resources' }.each {
-            it.attributes().remove('isTestSource')
-            it.attributes().put('type', 'java-resource')
-          }
-          it.asNode().component.content.sourceFolder.findAll { it.@url == 'file://$MODULE_DIR$/src/test/resources' }.each {
-            it.attributes().remove('isTestSource')
-            it.attributes().put('type', 'java-test-resource')
-          }
-        }
-      }
     }
   }
 
@@ -414,14 +391,6 @@ idea {
     vcs = 'Git'
   }
 }
-// Make sure gradle idea was run before running anything in intellij (including import).
-File ideaMarker = new File(projectDir, '.local-idea-is-configured')
-tasks.idea.doLast {
-  ideaMarker.setText('', 'UTF-8')
-}
-if (System.getProperty('idea.active') != null && ideaMarker.exists() == false) {
-  throw new GradleException('You must run `./gradlew idea` from the root of elasticsearch before importing into IntelliJ')
-}
 
 // eclipse configuration
 allprojects {

docs/reference/monitoring/configuring-metricbeat.asciidoc

@@ -81,53 +81,30 @@ run the following command:
 
 ["source","sh",subs="attributes,callouts"]
 ----------------------------------------------------------------------
-metricbeat modules enable elasticsearch
+metricbeat modules enable elasticsearch-xpack
 ----------------------------------------------------------------------
 
 For more information, see 
 {metricbeat-ref}/configuration-metricbeat.html[Specify which modules to run] and 
 {metricbeat-ref}/metricbeat-module-elasticsearch.html[{es} module]. 
 --
 
-.. Configure the {es} module in {metricbeat}. +
-+
---
-You must specify the following settings in the `modules.d/elasticsearch.yml` file:
-
-[source,yaml]
-----------------------------------
-- module: elasticsearch
-  metricsets:
-    - ccr
-    - cluster_stats
-    - index
-    - index_recovery
-    - index_summary
-    - ml_job
-    - node_stats
-    - shard
-  period: 10s
-  hosts: ["http://localhost:9200"] <1>
-  xpack.enabled: true <2>
-----------------------------------
-<1> This setting identifies the host and port number that are used to access {es}.
-<2> This setting ensures that {kib} can read this monitoring data successfully. 
-That is to say, it's stored in the same location and format as monitoring data 
-that is sent by <<es-monitoring-exporters,exporters>>. 
---
+.. By default the module will collect {es} monitoring metrics from `http://localhost:9200`.
+If the local {es} node has a different address, you must specify it via the `hosts` setting
+in the `modules.d/elasticsearch-xpack.yml` file.
 
-.. If Elastic {security-features} are enabled, you must also provide a user ID 
+.. If Elastic {security-features} are enabled, you must also provide a user ID
 and password so that {metricbeat} can collect metrics successfully. 
 
-... Create a user on the production cluster that has the 
+... Create a user on the production cluster that has the
 {stack-ov}/built-in-roles.html[`remote_monitoring_collector` built-in role]. 
 Alternatively, use the {stack-ov}/built-in-users.html[`remote_monitoring_user` built-in user].
 
-... Add the `username` and `password` settings to the {es} module configuration 
+... Add the `username` and `password` settings to the {es} module configuration
 file.
 +
 --
-For example, add the following settings in the `modules.d/elasticsearch.yml` file:
+For example, add the following settings in the `modules.d/elasticsearch-xpack.yml` file:
 
 [source,yaml]
 ----------------------------------
@@ -140,7 +117,7 @@ For example, add the following settings in the `modules.d/elasticsearch.yml` fil
 
 .. If you configured {es} to use <<configuring-tls,encrypted communications>>, 
 you must access it via HTTPS. For example, use a `hosts` setting like 
-`https://localhost:9200` in the `modules.d/elasticsearch.yml` file.
+`https://localhost:9200` in the `modules.d/elasticsearch-xpack.yml` file.
 
 .. Identify where to send the monitoring data. +
 +

qa/full-cluster-restart/build.gradle

@@ -105,3 +105,15 @@ task bwcTestSnapshots {
 
 check.dependsOn(bwcTestSnapshots)
 
+configurations {
+  testArtifacts.extendsFrom testRuntime
+}
+
+task testJar(type: Jar) {
+  appendix 'test'
+  from sourceSets.test.output
+}
+
+artifacts {
+  testArtifacts testJar
+}

server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java

@@ -147,7 +147,7 @@ public static void performOnPrimary(
         Consumer<ActionListener<Void>> waitForMappingUpdate,
         ActionListener<PrimaryResult<BulkShardRequest, BulkShardResponse>> listener,
         ThreadPool threadPool) {
-        new ActionRunnable<PrimaryResult<BulkShardRequest, BulkShardResponse>>(listener) {
+        new ActionRunnable<>(listener) {
 
             private final Executor executor = threadPool.executor(ThreadPool.Names.WRITE);
 
@@ -189,11 +189,11 @@ public void onRejection(Exception e) {
             }
 
             private void finishRequest() {
-                listener.onResponse(
-                    new WritePrimaryResult<>(context.getBulkShardRequest(), context.buildShardResponse(), context.getLocationToSync(),
-                        null, context.getPrimary(), logger));
+                ActionListener.completeWith(listener,
+                    () -> new WritePrimaryResult<>(
+                        context.getBulkShardRequest(), context.buildShardResponse(), context.getLocationToSync(), null,
+                        context.getPrimary(), logger));
             }
-
         }.run();
     }
 

server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java

@@ -254,7 +254,6 @@ public void testInvalidShardCountSettingsWithoutPrefix() throws Exception {
         }
     }
 
-    @AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/40951")
     public void testCreateAndDeleteIndexConcurrently() throws InterruptedException {
         createIndex("test");
         final AtomicInteger indexVersion = new AtomicInteger(0);

x-pack/plugin/core/build.gradle

@@ -48,6 +48,9 @@ dependencies {
     testCompile project(path: ':modules:reindex', configuration: 'runtime')
     testCompile project(path: ':modules:parent-join', configuration: 'runtime')
     testCompile project(path: ':modules:analysis-common', configuration: 'runtime')
+    testCompile(project(':x-pack:license-tools')) {
+        transitive = false
+    }
     testCompile ("org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}")
 }
 
@@ -95,8 +98,8 @@ licenseHeaders {
 }
 
 // make LicenseSigner available for testing signed licenses
-sourceSets.test.java {
-    srcDir '../../license-tools/src/main/java'
+sourceSets.test.resources {
+    srcDir 'src/main/config'
 }
 
 unitTest {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java

@@ -39,7 +39,7 @@ private class MockIndexer extends AsyncTwoPhaseIndexer<Integer, MockJobStats> {
 
         private final CountDownLatch latch;
         // test the execution order
-        private int step;
+        private volatile int step;
 
         protected MockIndexer(Executor executor, AtomicReference<IndexerState> initialState, Integer initialPosition,
                               CountDownLatch latch) {
@@ -113,8 +113,8 @@ protected void onFailure(Exception exc) {
         protected void onFinish(ActionListener<Void> listener) {
             assertThat(step, equalTo(4));
             ++step;
-            isFinished.set(true);
             listener.onResponse(null);
+            isFinished.set(true);
         }
 
         @Override
@@ -206,8 +206,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
     }
 
-    @AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/40946")
-    public void testStateMachine() throws InterruptedException {
+    public void testStateMachine() throws Exception {
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         final ExecutorService executor = Executors.newFixedThreadPool(1);
         isFinished.set(false);

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java

@@ -51,7 +51,6 @@ public void testNotCreatedWhenNoOtherMlIndices() {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/40933")
     public void testCreatedWhenAfterOtherMlIndex() throws Exception {
 
         Auditor auditor = new Auditor(client(), "node_1");

x-pack/plugin/security/build.gradle

@@ -136,6 +136,11 @@ dependencies {
 compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
 compileTestJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
 
+processTestResources {
+    from(project(xpackModule('core')).file('src/main/config'))
+    from(project(xpackModule('core')).file('src/test/resources'))
+}
+
 configurations {
     testArtifacts.extendsFrom testRuntime
 }
@@ -148,10 +153,7 @@ artifacts {
     archives jar
     testArtifacts testJar
 }
-sourceSets.test.resources {
-    srcDir '../core/src/test/resources'
-    srcDir '../core/src/main/config'
-}
+
 dependencyLicenses {
     mapping from: /java-support|opensaml-.*/, to: 'shibboleth'
     mapping from: /http.*/, to: 'httpclient'