Merge remote-tracking branch 'elastic/master' into cat-recovery-bytes

* elastic/master: (129 commits)
  Update ingest jdocs that a null return value will drop the current document. (elastic#40359)
  Remove -Xlint exclusions in the ingest-common module. (elastic#40505)
  Update docs for the DFR similarity (elastic#40579)
  Fix merging of search_as_you_type field mapper (elastic#40593)
  Support roles with application privileges against wildcard applications (elastic#40398)
  Remove obsolete security settings (elastic#40496)
  Remove Gradle deprecation warnings (elastic#40449)
  Run the build integ test in parallel (elastic#39788)
  Fix 3rd pary S3 tests (elastic#40588)
  lower bwc skip for search as you type (elastic#40599)
  Muting XContentParserTests#testSubParserArray
  Fixing typo in test error message (elastic#40611)
  Update max dims for vectors to 1024. (elastic#40597)
  Add start and stop time to cat recovery API (elastic#40378)
  [DOCS] Correct keystore commands for Email and Jira actions in Watcher (elastic#40417)
  [DOCS] Document common settings for snapshot repository plugins (elastic#40475)
  Remove with(out)-system-key tests (elastic#40547)
  Geo Point parse error fix (elastic#40447)
  Handle null retention leases in WaitForNoFollowersStep (elastic#40477)
  [DOCS] Adds anchors for ruby client (elastic#39867)
  ...

Author: jasontedor

.ci/init.gradle

@@ -0,0 +1,43 @@
+if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFACTORY_TOKEN == null) {
+    throw new GradleException("Using init script without configuration")
+} else {
+    logger.info("Using elastic artifactory repos")
+    settingsEvaluated { settings ->
+        settings.pluginManagement {
+            repositories {
+                maven {
+                    url "https://artifactory.elstc.co/artifactory/gradle-plugins"
+                    credentials {
+                        username System.env.ELASTIC_ARTIFACTORY_USERNAME
+                        password System.env.ELASTIC_ARTIFACTORY_TOKEN
+                    }
+                }
+                gradlePluginPortal()
+            }
+        }
+    }
+    projectsLoaded {
+        allprojects {
+            buildscript {
+                repositories {
+                    maven {
+                        url "https://artifactory.elstc.co/artifactory/gradle-release/"
+                        credentials {
+                            username System.env.ELASTIC_ARTIFACTORY_USERNAME
+                            password System.env.ELASTIC_ARTIFACTORY_TOKEN
+                        }
+                    }
+                }
+            }
+            repositories {
+                maven {
+                    url "https://artifactory.elstc.co/artifactory/gradle-release/"
+                    credentials {
+                        username System.env.ELASTIC_ARTIFACTORY_USERNAME
+                        password System.env.ELASTIC_ARTIFACTORY_TOKEN
+                    }
+                }
+            }
+        }
+    }
+}

.ci/matrix-runtime-javas.yml

@@ -9,6 +9,8 @@ ES_RUNTIME_JAVA:
   - java8
   - java8fips
   - java11
+  - java12
   - openjdk12
   - zulu8
   - zulu11
+  - zulu12

.ci/packer_cache.sh

@@ -20,5 +20,5 @@ export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA}
 # We are caching BWC versions too, need these so we can build those
 export JAVA8_HOME="${HOME}"/.java/java8
 export JAVA11_HOME="${HOME}"/.java/java11
-export JAVA12_HOME="${HOME}"/.java/java12
+export JAVA12_HOME="${HOME}"/.java/openjdk12
 ./gradlew --parallel clean --scan -Porg.elasticsearch.acceptScanTOS=true -s resolveAllDependencies

build.gradle

@@ -32,7 +32,7 @@ plugins {
     id 'com.gradle.build-scan' version '2.0.2'
     id 'base'
 }
-if (properties.get("org.elasticsearch.acceptScanTOS", "false") == "true") {
+if (Boolean.valueOf(project.findProperty('org.elasticsearch.acceptScanTOS') ?: "false")) {
     buildScan {
         termsOfServiceUrl = 'https://gradle.com/terms-of-service'
         termsOfServiceAgree = 'yes'
@@ -162,8 +162,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/40319" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/build.gradle

@@ -232,6 +232,11 @@ if (project != rootProject) {
     if (isLuceneSnapshot) {
       systemProperty 'test.lucene-snapshot-revision', isLuceneSnapshot[0][1]
     }
+    String defaultParallel = System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel)
+    if (defaultParallel == "auto") {
+        defaultParallel = Math.max(Runtime.getRuntime().availableProcessors(), 4)
+    }
+    maxParallelForks defaultParallel as Integer
   }
   check.dependsOn(integTest)
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -44,6 +44,7 @@ public class PluginBuildPlugin extends BuildPlugin {
     public void apply(Project project) {
         super.apply(project)
         configureDependencies(project)
+
         // this afterEvaluate must happen before the afterEvaluate added by integTest creation,
         // so that the file name resolution for installing the plugin will be setup
         project.afterEvaluate {
@@ -69,7 +70,7 @@ public class PluginBuildPlugin extends BuildPlugin {
                 if (isModule) {
                     throw new RuntimeException("Testclusters does not support modules yet");
                 } else {
-                    project.testClusters.integTestCluster.plugin(
+                    project.testClusters.integTest.plugin(
                             project.file(project.tasks.bundlePlugin.archiveFile)
                     )
                 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -693,10 +693,10 @@ class ClusterFormationTasks {
                 node.args.each { arg(value: it) }
                 if (Os.isFamily(Os.FAMILY_WINDOWS)) {
                     // Having no TMP on Windows defaults to C:\Windows and permission errors
-                    // Since we configure ant to run with a new  environment above, we need to set this to a dir we have access to
-                    File tmpDir = new File(node.baseDir, "tmp")
-                    tmpDir.mkdirs()
-                    env(key: "TMP", value: tmpDir.absolutePath)
+                    // Since we configure ant to run with a new  environment above, we need to explicitly pass this
+                    String tmp = System.getenv("TMP")
+                    assert tmp != null
+                    env(key: "TMP", value: tmp)
                 }
             }
         }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -62,13 +62,13 @@ public class RestIntegTestTask extends DefaultTask {
             clusterConfig = project.extensions.create("${name}Cluster", ClusterConfiguration.class, project)
         } else {
             project.testClusters {
-                integTestCluster {
+                "$name" {
                     distribution = 'INTEG_TEST'
                     version = project.version
                     javaHome = project.file(project.ext.runtimeJavaHome)
                 }
             }
-            runner.useCluster project.testClusters.integTestCluster
+            runner.useCluster project.testClusters."$name"
         }
 
         // override/add more for rest tests
@@ -81,7 +81,7 @@ public class RestIntegTestTask extends DefaultTask {
                 throw new IllegalArgumentException("tests.rest.cluster and tests.cluster must both be null or non-null")
             }
             if (usesTestclusters == true) {
-                ElasticsearchNode node = project.testClusters.integTestCluster
+                ElasticsearchNode node = project.testClusters."${name}"
                 runner.systemProperty('tests.rest.cluster', {node.allHttpSocketURI.join(",") })
                 runner.systemProperty('tests.config.dir', {node.getConfigDir()})
                 runner.systemProperty('tests.cluster', {node.transportPortURI})
@@ -187,6 +187,10 @@ public class RestIntegTestTask extends DefaultTask {
         clusterInit.mustRunAfter(tasks)
     }
 
+    public void runner(Closure configure) {
+        project.tasks.getByName("${name}Runner").configure(configure)
+    }
+
     /** Print out an excerpt of the log from the given node. */
     protected static void printLogExcerpt(NodeInfo nodeInfo) {
         File logFile = new File(nodeInfo.homeDir, "logs/${nodeInfo.clusterName}.log")

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -78,7 +78,6 @@ class VagrantTestPlugin implements Plugin<Project> {
     private static final PACKAGING_TEST_CONFIGURATION = 'packagingTest'
     private static final BATS = 'bats'
     private static final String BATS_TEST_COMMAND ="cd \$PACKAGING_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS"
-    private static final String PLATFORM_TEST_COMMAND ="rm -rf ~/elasticsearch && rsync -r /elasticsearch/ ~/elasticsearch && cd ~/elasticsearch && ./gradlew test integTest"
 
     /** Boxes that have been supplied and are available for testing **/
     List<String> availableBoxes = []
@@ -388,15 +387,6 @@ class VagrantTestPlugin implements Plugin<Project> {
         }
     }
 
-    private static void createPlatformTestTask(Project project) {
-        project.tasks.create('platformTest') {
-            group 'Verification'
-            description "Test unit and integ tests on different platforms using vagrant. See TESTING.asciidoc for details. This test " +
-                    "is unmaintained."
-            dependsOn 'vagrantCheckVersion'
-        }
-    }
-
     private void createBoxListTasks(Project project) {
         project.tasks.create('listAllBoxes') {
             group 'Verification'
@@ -429,7 +419,6 @@ class VagrantTestPlugin implements Plugin<Project> {
         createSmokeTestTask(project)
         createPrepareVagrantTestEnvTask(project)
         createPackagingTestTask(project)
-        createPlatformTestTask(project)
         createBoxListTasks(project)
     }
 
@@ -454,9 +443,6 @@ class VagrantTestPlugin implements Plugin<Project> {
         assert project.tasks.packagingTest != null
         Task packagingTest = project.tasks.packagingTest
 
-        assert project.tasks.platformTest != null
-        Task platformTest = project.tasks.platformTest
-
         /*
          * We always use the main project.rootDir as Vagrant's current working directory (VAGRANT_CWD)
          * so that boxes are not duplicated for every Gradle project that use this VagrantTestPlugin.
@@ -610,31 +596,6 @@ class VagrantTestPlugin implements Plugin<Project> {
                     packagingTest.dependsOn(javaPackagingTest)
                 }
             }
-
-            /*
-             * This test is unmaintained and was created to run on Linux. We won't allow it to run on Windows
-             * until it's been brought back into maintenance
-             */
-            if (LINUX_BOXES.contains(box)) {
-                Task platform = project.tasks.create("vagrant${boxTask}#platformTest", VagrantCommandTask) {
-                    command 'ssh'
-                    boxName box
-                    environmentVars vagrantEnvVars
-                    dependsOn up
-                    finalizedBy halt
-                    args '--command', PLATFORM_TEST_COMMAND + " -Dtests.seed=${-> project.testSeed}"
-                }
-                TaskExecutionAdapter platformReproListener = createReproListener(project, platform.path)
-                platform.doFirst {
-                    project.gradle.addListener(platformReproListener)
-                }
-                platform.doLast {
-                    project.gradle.removeListener(platformReproListener)
-                }
-                if (project.extensions.esvagrant.boxes.contains(box)) {
-                    platformTest.dependsOn(platform)
-                }
-            }
         }
     }
 

buildSrc/src/main/java/org/elasticsearch/GradleServicesAdapter.java

@@ -30,6 +30,7 @@
 import org.gradle.process.JavaExecSpec;
 
 import java.io.File;
+import java.nio.file.Path;
 
 /**
  * Bridge a gap until Gradle offers service injection for plugins.
@@ -76,4 +77,8 @@ public FileCollection fileTree(File dir) {
     public void loggedExec(Action<ExecSpec> action) {
         LoggedExec.exec(project, action);
     }
+
+    public void delete(Path path) {
+        project.delete(path.toFile());
+    }
 }