Merge branch '6.x' into ccr-6.x

* 6.x:
  Test: Do not remove xpack templates when cleaning (#31642)
  SQL: Allow long literals (#31777)
  SQL: Fix incorrect message for aliases (#31792)
  Detach Transport from TransportService (#31727)
  6.3.1 release notes (#31829)
  Add unreleased version 6.3.2
  [ML][TEST] Use java 11 valid time format in DataDescriptionTests (#31817)
  [ML] Don't treat stale FAILED jobs as OPENING in job allocation (#31800)
  [ML] Fix calendar and filter updates from non-master nodes (#31804)
  Fix license header generation on Windows (#31790)
  mark XPackRestIT.test {p0=monitoring/bulk/10_basic/Bulk indexing of monitoring data} as AwaitsFix
  Add JDK11 support without enabling in CI (#31644)
  Watcher: Fix check for currently executed watches (#31137)
  [DOCS] Fixes 6.3.0 release notes (#31771)
  Watcher: Ensure correct method is used to read secure settings (#31753)
  [ML] Rate limit established model memory updates (#31768)
  SQL: Update CLI logo

Author: dnhatn

build.gradle

@@ -439,12 +439,19 @@ allprojects {
   }
 
   File licenseHeaderFile;
-  if (eclipse.project.name.startsWith(':x-pack')) {
+  String prefix = ':x-pack';
+ 
+  if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+    prefix = prefix.replace(':', '_')
+  }
+  if (eclipse.project.name.startsWith(prefix)) {
       licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/elastic-license-header.txt')
   } else {
       licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/oss-license-header.txt')
   }
-  String licenseHeader = licenseHeaderFile.getText('UTF-8').replace('\n', '\\\\n')
+  
+  String lineSeparator = Os.isFamily(Os.FAMILY_WINDOWS) ? '\\\\r\\\\n' : '\\\\n'
+  String licenseHeader = licenseHeaderFile.getText('UTF-8').replace(System.lineSeparator(), lineSeparator)
   task copyEclipseSettings(type: Copy) {
     // TODO: "package this up" for external builds
     from new File(project.rootDir, 'buildSrc/src/main/resources/eclipse.settings')

buildSrc/src/main/resources/forbidden/jdk-signatures.txt

@@ -88,7 +88,8 @@ java.lang.Thread#getAllStackTraces()
 
 @defaultMessage Stopping threads explicitly leads to inconsistent states. Use interrupt() instead.
 java.lang.Thread#stop()
-java.lang.Thread#stop(java.lang.Throwable)
+# uncomment when https://github.com/elastic/elasticsearch/issues/31715 is fixed
+# java.lang.Thread#stop(java.lang.Throwable)
 
 @defaultMessage Please do not terminate the application
 java.lang.System#exit(int)

distribution/bwc/build.gradle

@@ -18,12 +18,14 @@
  */
 
 
+
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 
-import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
+import java.nio.charset.StandardCharsets
 
+import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
 /**
  * This is a dummy project which does a local checkout of the previous
  * version's branch, and builds a snapshot. This allows backcompat
@@ -147,12 +149,16 @@ subprojects {
 
   task buildBwcVersion(type: Exec) {
     dependsOn checkoutBwcBranch, writeBuildMetadata
+    // send RUNTIME_JAVA_HOME so the build doesn't fails on newer version the branch doesn't know about
+    environment('RUNTIME_JAVA_HOME', getJavaHome(it, rootProject.ext.minimumRuntimeVersion.getMajorVersion() as int))
     workingDir = checkoutDir
+    // we are building branches that are officially built with JDK 8, push JAVA8_HOME to JAVA_HOME for these builds
     if (["5.6", "6.0", "6.1"].contains(bwcBranch)) {
-      // we are building branches that are officially built with JDK 8, push JAVA8_HOME to JAVA_HOME for these builds
       environment('JAVA_HOME', getJavaHome(it, 8))
     } else if ("6.2".equals(bwcBranch)) {
       environment('JAVA_HOME', getJavaHome(it, 9))
+    } else if (["6.3", "6.x"].contains(bwcBranch)) {
+      environment('JAVA_HOME', getJavaHome(it, 10))
     } else {
       environment('JAVA_HOME', project.compilerJavaHome)
     }
@@ -177,6 +183,8 @@ subprojects {
     } else if (showStacktraceName.equals("ALWAYS_FULL")) {
       args "--full-stacktrace"
     }
+    standardOutput = new IndentingOutputStream(System.out)
+    errorOutput = new IndentingOutputStream(System.err)
     doLast {
       List missing = artifactFiles.grep { file ->
         false == file.exists()
@@ -196,3 +204,27 @@ subprojects {
     }
   }
 }
+
+class IndentingOutputStream extends OutputStream {
+
+  public static final byte[] INDENT = "  [bwc] ".getBytes(StandardCharsets.UTF_8)
+  private final OutputStream delegate
+
+  public IndentingOutputStream(OutputStream delegate) {
+      this.delegate = delegate
+  }
+
+  @Override
+  public void write(int b) {
+    write([b] as int[], 0, 1)
+  }
+
+  public void write(int[] bytes, int offset, int length) {
+    for (int i = 0; i < bytes.length; i++) {
+      delegate.write(bytes[i])
+      if (bytes[i] == '\n') {
+        delegate.write(INDENT)
+      }
+    }
+  }
+}

docs/build.gradle

@@ -39,6 +39,15 @@ integTestCluster {
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
 }
 
+// remove when https://github.com/elastic/elasticsearch/issues/31305 is fixed
+if (rootProject.ext.compilerJavaVersion.isJava11()) {
+  integTestRunner {
+    systemProperty 'tests.rest.blacklist', [
+            'plugins/ingest-attachment/line_164',
+            'plugins/ingest-attachment/line_117'
+    ].join(',')
+  }
+}
 // Build the cluster with all plugins
 
 project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->

docs/reference/release-notes/6.3.asciidoc

@@ -1,7 +1,57 @@
 [[release-notes-6.3.1]]
 == {es} version 6.3.1
 
-coming[6.3.1]
+Also see <<breaking-changes-6.3>>.
+
+[[bug-6.3.1]]
+[float]
+=== Bug fixes
+
+Authentication::
+* Security: fix joining cluster with production license {pull}31341[#31341] (issue: {issue}31332[#31332])
+* Security: fix token bwc with pre 6.0.0-beta2 {pull}31254[#31254] (issues: {issue}30743[#30743], {issue}31195[#31195])
+* Compliant SAML Response destination check {pull}31175[#31175]
+
+Ingest::
+* Ingest Attachment: Upgrade Tika to 1.18 {pull}31252[#31252]
+
+Java High Level REST Client::
+* Change bulk's retry condition to be based on RestStatus {pull}29329[#29329] (issues: {issue}28885[#28885], {issue}29254[#29254])
+
+Java Low Level REST Client::
+* Avoid setting connection request timeout {pull}30384[#30384] (issue: {issue}24069[#24069])
+
+Machine Learning::
+* Fixes a bug introduced in 6.3.0 which may cause the a node to hang and drop out of the cluster if Machine Learning is enabled and has been actively used. The issue can be redressed in 6.3.0 by disabling Machine Learning on all nodes {pull}31691[#31691] (issue: {issue}31683[#31683])
+
+Network::
+* Ensure we don't use a remote profile if cluster name matches {pull}31331[#31331] (issue: {issue}29321[#29321])
+
+Packaging::
+* Add package pre-install check for java binary {pull}31343[#31343] (issue: {issue}29665[#29665])
+
+Recovery::
+* Fix missing historyUUID in peer recovery when rolling upgrade 5.x to 6.3 {pull}31506[#31506] (issue: {issue}31482[#31482])
+
+Rollup::
+* [Rollup] Metric config parser must use builder so validation runs {pull}31159[#31159]
+
+SQL::
+* JDBC: Fix stackoverflow on getObject and timestamp conversion {pull}31735[#31735] (issue: {issue}31734[#31734])
+* SQL: Preserve scoring in bool queries {pull}30730[#30730] (issue: {issue}29685[#29685])
+
+Scripting::
+* Painless: Fix bug for static method calls on interfaces {pull}31348[#31348]
+
+Search::
+* Fix race in clear scroll {pull}31259[#31259]
+* Cross Cluster Search: preserve remote status code {pull}30976[#30976] (issue: {issue}27461[#27461])
+
+Security::
+* Preserve thread context when connecting to remote cluster {pull}31574[#31574] (issues: {issue}31241[#31241], {issue}31462[#31462])
+
+Watcher::
+* Watcher: Fix put watch action {pull}31524[#31524]
 
 [[release-notes-6.3.0]]
 == {es} version 6.3.0
@@ -93,9 +143,6 @@ Aggregations::
 * Adds the ability to specify a format on composite date_histogram source {pull}28310[#28310] (issue: {issue}27923[#27923])
 * Calculate sum in Kahan summation algorithm in aggregations (#27807) {pull}27848[#27848] (issue: {issue}27807[#27807])
 
-Discovery-Plugins::
-* Update ec2 secure settings {pull}29134[#29134]
-
 Geo::
 * Add Z value support to geo_point and geo_shape {pull}25738[#25738] (issue: {issue}22917[#22917])
 
@@ -124,9 +171,6 @@ Scripting::
 Search::
 * Search - new flag: allow_partial_search_results {pull}27906[#27906] (issue: {issue}27435[#27435])
 
-Snapshot/Restore::
-* Update s3 secure settings {pull}28517[#28517]
-
 Task Management::
 * Add new setting to disable persistent tasks allocations {pull}29137[#29137]
 
@@ -403,7 +447,7 @@ License::
 * Do not serialize basic license exp in x-pack info {pull}30848[#30848]
 *  Require acknowledgement to start_trial license {pull}30198[#30198] (issue: {issue}30134[#30134])
 
-Machine Learning:: 
+Machine Learning::
 * By-fields should respect model_plot_config.terms {ml-pull}86[#86] (issue: {issue}30004[#30004])
 * Function description for population lat_long results should be lat_long instead of mean {ml-pull}81[#81] (issue: {ml-issue}80[#80])
 * Fix error causing us to overestimate effective history length {ml-pull}66[#66] (issue: {ml-issue}57[#57])

modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.bootstrap.JavaVersion;
 import org.hamcrest.Matcher;
 
 import java.lang.invoke.MethodHandle;
@@ -41,7 +42,11 @@ protected String valueCtorCall(String valueType, int size) {
 
     @Override
     protected Matcher<String> outOfBoundsExceptionMessageMatcher(int index, int size) {
-        return equalTo(Integer.toString(index));
+        if (JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0) {
+            return equalTo(Integer.toString(index));
+        } else{
+            return equalTo("Index " + Integer.toString(index) + " out of bounds for length " + Integer.toString(size));
+        }
     }
 
     public void testArrayLengthHelper() throws Throwable {

modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java

@@ -88,7 +88,7 @@ public void testScheduledPing() throws Exception {
         assertThat(nettyA.getPing().getFailedPings(), equalTo(0L));
         assertThat(nettyB.getPing().getFailedPings(), equalTo(0L));
 
-        serviceA.registerRequestHandler("sayHello", TransportRequest.Empty::new, ThreadPool.Names.GENERIC,
+        serviceA.registerRequestHandler("internal:sayHello", TransportRequest.Empty::new, ThreadPool.Names.GENERIC,
             new TransportRequestHandler<TransportRequest.Empty>() {
                 @Override
                 public void messageReceived(TransportRequest.Empty request, TransportChannel channel) {
@@ -103,7 +103,7 @@ public void messageReceived(TransportRequest.Empty request, TransportChannel cha
 
         int rounds = scaledRandomIntBetween(100, 5000);
         for (int i = 0; i < rounds; i++) {
-            serviceB.submitRequest(nodeA, "sayHello",
+            serviceB.submitRequest(nodeA, "internal:sayHello",
                 TransportRequest.Empty.INSTANCE, TransportRequestOptions.builder().withCompress(randomBoolean()).build(),
                 new TransportResponseHandler<TransportResponse.Empty>() {
                     @Override

plugins/ingest-attachment/build.gradle

@@ -25,11 +25,21 @@ esplugin {
 versions << [
   'tika': '1.18',
   'pdfbox': '2.0.9',
-  'bouncycastle': '1.55',
+  'bouncycastle': '1.59',
   'poi': '3.17',
   'mime4j': '0.8.1'
 ]
 
+if (rootProject.ext.compilerJavaVersion.isJava11()) {
+  // disabled until https://github.com/elastic/elasticsearch/issues/31456 is fixed.
+  integTestRunner {
+    systemProperty 'tests.rest.blacklist', [
+            'ingest_attachment/20_attachment_processor/Test indexed chars are configurable',
+            'ingest_attachment/20_attachment_processor/Test indexed chars are configurable per document'
+    ].join(',')
+  }
+}
+
 dependencies {
   // mandatory for tika
   compile "org.apache.tika:tika-core:${versions.tika}"

plugins/ingest-attachment/licenses/bcmail-jdk15on-1.55.jar.sha1

@@ -0,0 +1 @@
+db389ade95f48592908a84e7050a691c8834723c