Merge branch 'master' into translog-generation

* master:
  [API] change wait_for_completion defaults according to docs (elastic#23672)
  Share XContent rendering code in terms aggs (elastic#23680)
  Update ingest-node.asciidoc
  [DOCS] Update the docs about the fact that global ordinals for _parent field are loaded eagerly instead of lazily by default.
  Build: remove progress logger hack for gradle 2.13 (elastic#23679)
  Test: Add dump of integ test cluster logs on failure (elastic#23688)
  Plugins: Add plugin cli specific exit codes (elastic#23599)
  Plugins: Output better error message when existing plugin is incompatible (elastic#23562)
  Reindex: wait for cleanup before responding (elastic#23677)
  Packaging: Remove classpath ordering hack (elastic#23596)
  Docs: Add note about updating plugins requiring removal and reinstallation (elastic#23597)
  Build: Make plugin list for smoke tester dynamic (elastic#23601)
  [TEST] Propertly cleans up failing restore test

Author: jasontedor

buildSrc/build.gradle

@@ -96,23 +96,12 @@ dependencies {
   compile 'org.apache.rat:apache-rat:0.11'
 }
 
-// Gradle version-specific options (allows build to run with Gradle 2.13 as well as 2.14+/3.+)
-if (GradleVersion.current() == GradleVersion.version("2.13")) {
-  // ProgressLogger(-Factory) classes are part of the public Gradle API
-  sourceSets.main.groovy.srcDir 'src/main/gradle-2.13-groovy'
+// Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs
+// Use logging dependency instead
 
-  dependencies {
-    compile 'ru.vyarus:gradle-animalsniffer-plugin:1.0.1' // last version compatible with Gradle 2.13
-  }
-} else {
-  // Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs
-  // Use logging dependency instead
-  sourceSets.main.groovy.srcDir 'src/main/gradle-2.14-groovy'
-
-  dependencies {
-    compileOnly "org.gradle:gradle-logging:${GradleVersion.current().getVersion()}"
-    compile 'ru.vyarus:gradle-animalsniffer-plugin:1.2.0' // Gradle 2.14 requires a version > 1.0.1
-  }
+dependencies {
+  compileOnly "org.gradle:gradle-logging:${GradleVersion.current().getVersion()}"
+  compile 'ru.vyarus:gradle-animalsniffer-plugin:1.2.0' // Gradle 2.14 requires a version > 1.0.1
 }
 
 /*****************************************************************************

buildSrc/src/main/gradle-2.13-groovy/org/elasticsearch/gradle/ProgressLogger.groovy

@@ -8,7 +8,6 @@ import org.apache.tools.ant.BuildException
 import org.apache.tools.ant.DefaultLogger
 import org.apache.tools.ant.RuntimeConfigurable
 import org.apache.tools.ant.UnknownElement
-import org.elasticsearch.gradle.ProgressLoggerFactoryInjection
 import org.gradle.api.DefaultTask
 import org.gradle.api.file.FileCollection
 import org.gradle.api.file.FileTreeElement
@@ -20,9 +19,12 @@ import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.util.PatternFilterable
 import org.gradle.api.tasks.util.PatternSet
+import org.gradle.internal.logging.progress.ProgressLoggerFactory
 import org.gradle.util.ConfigureUtil
 
-class RandomizedTestingTask extends DefaultTask implements ProgressLoggerFactoryInjection {
+import javax.inject.Inject
+
+class RandomizedTestingTask extends DefaultTask {
 
     // TODO: change to "executable" to match gradle test params?
     @Optional
@@ -92,6 +94,11 @@ class RandomizedTestingTask extends DefaultTask implements ProgressLoggerFactory
         listenersConfig.listeners.add(new TestReportLogger(logger: logger, config: testLoggingConfig))
     }
 
+    @Inject
+    ProgressLoggerFactory getProgressLoggerFactory() {
+        throw new UnsupportedOperationException()
+    }
+
     void jvmArgs(Iterable<String> arguments) {
         jvmArgs.addAll(arguments)
     }

buildSrc/src/main/gradle-2.13-groovy/org/elasticsearch/gradle/ProgressLoggerFactoryInjection.groovy

@@ -25,7 +25,8 @@ import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedStartEvent
 import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedSuiteResultEvent
 import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedTestResultEvent
 import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener
-import org.elasticsearch.gradle.ProgressLogger
+import org.gradle.internal.logging.progress.ProgressLogger
+import org.gradle.internal.logging.progress.ProgressLoggerFactory
 
 import static com.carrotsearch.ant.tasks.junit4.FormattingUtils.formatDurationInSeconds
 import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.ERROR
@@ -51,6 +52,8 @@ import static java.lang.Math.max
  * quick.
  */
 class TestProgressLogger implements AggregatedEventListener {
+    /** Factory to build a progress logger when testing starts */
+    ProgressLoggerFactory factory
     ProgressLogger progressLogger
     int totalSuites
     int totalSlaves
@@ -74,17 +77,14 @@ class TestProgressLogger implements AggregatedEventListener {
     /** Have we finished a whole suite yet? */
     volatile boolean suiteFinished = false
     /* Note that we probably overuse volatile here but it isn't hurting us and
-      lets us move things around without worying about breaking things. */
-
-    TestProgressLogger(Map args) {
-        progressLogger = new ProgressLogger(args.factory.newOperation(TestProgressLogger))
-        progressLogger.setDescription('Randomized test runner')
-    }
+       lets us move things around without worrying about breaking things. */
 
     @Subscribe
     void onStart(AggregatedStartEvent e) throws IOException {
         totalSuites = e.suiteCount
         totalSlaves = e.slaveCount
+        progressLogger = factory.newOperation(TestProgressLogger)
+        progressLogger.setDescription('Randomized test runner')
         progressLogger.started()
         progressLogger.progress(
             "Starting JUnit4 for ${totalSuites} suites on ${totalSlaves} jvms")

buildSrc/src/main/gradle-2.14-groovy/org/elasticsearch/gradle/ProgressLogger.groovy

@@ -22,10 +22,15 @@ import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import org.elasticsearch.gradle.BuildPlugin
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
+import org.gradle.api.execution.TaskExecutionAdapter
 import org.gradle.api.internal.tasks.options.Option
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.tasks.Input
-import org.gradle.util.ConfigureUtil
+import org.gradle.api.tasks.TaskState
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.util.stream.Stream
 
 /**
  * A wrapper task around setting up a cluster and running rest tests.
@@ -71,6 +76,24 @@ public class RestIntegTestTask extends DefaultTask {
         // both as separate sysprops
         runner.systemProperty('tests.cluster', "${-> nodes[0].transportUri()}")
 
+        // dump errors and warnings from cluster log on failure
+        TaskExecutionAdapter logDumpListener = new TaskExecutionAdapter() {
+            @Override
+            void afterExecute(Task task, TaskState state) {
+                if (state.failure != null) {
+                    for (NodeInfo nodeInfo : nodes) {
+                        printLogExcerpt(nodeInfo)
+                    }
+                }
+            }
+        }
+        runner.doFirst {
+            project.gradle.addListener(logDumpListener)
+        }
+        runner.doLast {
+            project.gradle.removeListener(logDumpListener)
+        }
+
         // copy the rest spec/tests into the test resources
         RestSpecHack.configureDependencies(project)
         project.afterEvaluate {
@@ -126,4 +149,42 @@ public class RestIntegTestTask extends DefaultTask {
     public Task mustRunAfter(Object... tasks) {
         clusterInit.mustRunAfter(tasks)
     }
+
+    /** Print out an excerpt of the log from the given node. */
+    protected static void printLogExcerpt(NodeInfo nodeInfo) {
+        File logFile = new File(nodeInfo.homeDir, "logs/${nodeInfo.clusterName}.log")
+        println("\nCluster ${nodeInfo.clusterName} - node ${nodeInfo.nodeNum} log excerpt:")
+        println("(full log at ${logFile})")
+        println('-----------------------------------------')
+        Stream<String> stream = Files.lines(logFile.toPath(), StandardCharsets.UTF_8)
+        try {
+            boolean inStartup = true
+            boolean inExcerpt = false
+            int linesSkipped = 0
+            for (String line : stream) {
+                if (line.startsWith("[")) {
+                    inExcerpt = false // clear with the next log message
+                }
+                if (line =~ /(\[WARN\])|(\[ERROR\])/) {
+                    inExcerpt = true // show warnings and errors
+                }
+                if (inStartup || inExcerpt) {
+                    if (linesSkipped != 0) {
+                        println("... SKIPPED ${linesSkipped} LINES ...")
+                    }
+                    println(line)
+                    linesSkipped = 0
+                } else {
+                    ++linesSkipped
+                }
+                if (line =~ /recovered \[\d+\] indices into cluster_state/) {
+                    inStartup = false
+                }
+            }
+        } finally {
+            stream.close()
+        }
+        println('=========================================')
+
+    }
 }

buildSrc/src/main/gradle-2.14-groovy/org/elasticsearch/gradle/ProgressLoggerFactoryInjection.groovy

@@ -19,10 +19,9 @@
 package org.elasticsearch.gradle.vagrant
 
 import com.carrotsearch.gradle.junit4.LoggingOutputStream
-import groovy.transform.PackageScope
-import org.elasticsearch.gradle.ProgressLogger
 import org.gradle.api.GradleScriptException
 import org.gradle.api.logging.Logger
+import org.gradle.internal.logging.progress.ProgressLogger
 
 import java.util.regex.Matcher
 
@@ -48,7 +47,7 @@ public class TapLoggerOutputStream extends LoggingOutputStream {
 
     TapLoggerOutputStream(Map args) {
         logger = args.logger
-        progressLogger = new ProgressLogger(args.factory.newOperation(VagrantLoggerOutputStream))
+        progressLogger = args.factory.newOperation(VagrantLoggerOutputStream)
         progressLogger.setDescription("TAP output for `${args.command}`")
     }
 

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy

@@ -19,15 +19,17 @@
 package org.elasticsearch.gradle.vagrant
 
 import org.apache.commons.io.output.TeeOutputStream
-import org.elasticsearch.gradle.ProgressLoggerFactoryInjection
 import org.elasticsearch.gradle.LoggedExec
 import org.gradle.api.tasks.Input
+import org.gradle.internal.logging.progress.ProgressLoggerFactory
+
+import javax.inject.Inject
 
 /**
  * Runs a vagrant command. Pretty much like Exec task but with a nicer output
  * formatter and defaults to `vagrant` as first part of commandLine.
  */
-public class VagrantCommandTask extends LoggedExec implements ProgressLoggerFactoryInjection {
+public class VagrantCommandTask extends LoggedExec {
 
     @Input
     String boxName
@@ -47,6 +49,11 @@ public class VagrantCommandTask extends LoggedExec implements ProgressLoggerFact
         }
     }
 
+    @Inject
+    ProgressLoggerFactory getProgressLoggerFactory() {
+        throw new UnsupportedOperationException()
+    }
+
     protected OutputStream createLoggerOutputStream() {
         return new VagrantLoggerOutputStream(
             command: commandLine.join(' '),