Add JDK11 support without enabling in CI (elastic#31644)

* Upgrade bouncycastle

Required to fix
`bcprov-jdk15on-1.55.jar; invalid manifest format `
on jdk 11

* Downgrade bouncycastle to avoid invalid manifest

* Add checksum for new jars

* Update tika permissions for jdk 11

* Mute test failing on jdk 11

* Add JDK11 to CI

* Thread#stop(Throwable) was removed

http://mail.openjdk.java.net/pipermail/core-libs-dev/2018-June/053536.html

* Disable failing tests elastic#31456

* Temprorarily disable doc tests

To see if there are other failures on JDK11

* Only blacklist specific doc tests

* Disable only failing tests in ingest attachment plugin

* Mute failing HDFS tests elastic#31498

* Mute failing lang-painless tests elastic#31500

* Fix backwards compatability builds

Fix JAVA version to 10 for ES 6.3

* Add 6.x to bwx -> java10

* Prefix out and err from buildBwcVersion for readability

```
> Task :distribution:bwc:next-bugfix-snapshot:buildBwcVersion
  [bwc] :buildSrc:compileJava
  [bwc] WARNING: An illegal reflective access operation has occurred
  [bwc] WARNING: Illegal reflective access by org.codehaus.groovy.reflection.CachedClass (file:/home/alpar/.gradle/wrapper/dists/gradle-4.5-all/cg9lyzfg3iwv6fa00os9gcgj4/gradle-4.5/lib/groovy-all-2.4.12.jar) to method java.lang.Object.finalize()
  [bwc] WARNING: Please consider reporting this to the maintainers of org.codehaus.groovy.reflection.CachedClass
  [bwc] WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
  [bwc] WARNING: All illegal access operations will be denied in a future release
  [bwc] :buildSrc:compileGroovy
  [bwc] :buildSrc:writeVersionProperties
  [bwc] :buildSrc:processResources
  [bwc] :buildSrc:classes
  [bwc] :buildSrc:jar

```

* Also set RUNTIME_JAVA_HOME for bwcBuild

So that we can make sure it's not too new for the build to understand.

* Align bouncycastle dependency

* fix painles array tets

closes elastic#31500

* Update jar checksums

* Keep 8/10 runtime/compile untill consensus builds on 11

* Only skip failing tests if running on Java 11

* Failures are dependent of compile java version not runtime

* Condition doc test exceptions on compiler java version as well

* Disable hdfs tests based on runtime java

* Set runtime java to minimum supported for bwc

* PR review

* Add comment with ticket for forbidden apis

Author: alpar-t

buildSrc/src/main/resources/forbidden/jdk-signatures.txt

@@ -88,7 +88,8 @@ java.lang.Thread#getAllStackTraces()
 
 @defaultMessage Stopping threads explicitly leads to inconsistent states. Use interrupt() instead.
 java.lang.Thread#stop()
-java.lang.Thread#stop(java.lang.Throwable)
+# uncomment when https://github.com/elastic/elasticsearch/issues/31715 is fixed
+# java.lang.Thread#stop(java.lang.Throwable)
 
 @defaultMessage Please do not terminate the application
 java.lang.System#exit(int)

distribution/bwc/build.gradle

@@ -18,12 +18,14 @@
  */
 
 
+
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 
-import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
+import java.nio.charset.StandardCharsets
 
+import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
 /**
  * This is a dummy project which does a local checkout of the previous
  * version's branch, and builds a snapshot. This allows backcompat
@@ -147,12 +149,16 @@ subprojects {
 
   task buildBwcVersion(type: Exec) {
     dependsOn checkoutBwcBranch, writeBuildMetadata
+    // send RUNTIME_JAVA_HOME so the build doesn't fails on newer version the branch doesn't know about
+    environment('RUNTIME_JAVA_HOME', getJavaHome(it, rootProject.ext.minimumRuntimeVersion.getMajorVersion() as int))
     workingDir = checkoutDir
+    // we are building branches that are officially built with JDK 8, push JAVA8_HOME to JAVA_HOME for these builds
     if (["5.6", "6.0", "6.1"].contains(bwcBranch)) {
-      // we are building branches that are officially built with JDK 8, push JAVA8_HOME to JAVA_HOME for these builds
       environment('JAVA_HOME', getJavaHome(it, 8))
     } else if ("6.2".equals(bwcBranch)) {
       environment('JAVA_HOME', getJavaHome(it, 9))
+    } else if (["6.3", "6.x"].contains(bwcBranch)) {
+      environment('JAVA_HOME', getJavaHome(it, 10))
     } else {
       environment('JAVA_HOME', project.compilerJavaHome)
     }
@@ -177,6 +183,8 @@ subprojects {
     } else if (showStacktraceName.equals("ALWAYS_FULL")) {
       args "--full-stacktrace"
     }
+    standardOutput = new IndentingOutputStream(System.out)
+    errorOutput = new IndentingOutputStream(System.err)
     doLast {
       List missing = artifactFiles.grep { file ->
         false == file.exists()
@@ -196,3 +204,27 @@ subprojects {
     }
   }
 }
+
+class IndentingOutputStream extends OutputStream {
+
+  public static final byte[] INDENT = "  [bwc] ".getBytes(StandardCharsets.UTF_8)
+  private final OutputStream delegate
+
+  public IndentingOutputStream(OutputStream delegate) {
+      this.delegate = delegate
+  }
+
+  @Override
+  public void write(int b) {
+    write([b] as int[], 0, 1)
+  }
+
+  public void write(int[] bytes, int offset, int length) {
+    for (int i = 0; i < bytes.length; i++) {
+      delegate.write(bytes[i])
+      if (bytes[i] == '\n') {
+        delegate.write(INDENT)
+      }
+    }
+  }
+}

docs/build.gradle

@@ -39,6 +39,15 @@ integTestCluster {
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
 }
 
+// remove when https://github.com/elastic/elasticsearch/issues/31305 is fixed
+if (rootProject.ext.compilerJavaVersion.isJava11()) {
+  integTestRunner {
+    systemProperty 'tests.rest.blacklist', [
+            'plugins/ingest-attachment/line_164',
+            'plugins/ingest-attachment/line_117'
+    ].join(',')
+  }
+}
 // Build the cluster with all plugins
 
 project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->

modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.bootstrap.JavaVersion;
 import org.hamcrest.Matcher;
 
 import java.lang.invoke.MethodHandle;
@@ -41,7 +42,11 @@ protected String valueCtorCall(String valueType, int size) {
 
     @Override
     protected Matcher<String> outOfBoundsExceptionMessageMatcher(int index, int size) {
-        return equalTo(Integer.toString(index));
+        if (JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0) {
+            return equalTo(Integer.toString(index));
+        } else{
+            return equalTo("Index " + Integer.toString(index) + " out of bounds for length " + Integer.toString(size));
+        }
     }
 
     public void testArrayLengthHelper() throws Throwable {

plugins/ingest-attachment/build.gradle

@@ -25,11 +25,21 @@ esplugin {
 versions << [
   'tika': '1.18',
   'pdfbox': '2.0.9',
-  'bouncycastle': '1.55',
+  'bouncycastle': '1.59',
   'poi': '3.17',
   'mime4j': '0.8.1'
 ]
 
+if (rootProject.ext.compilerJavaVersion.isJava11()) {
+  // disabled until https://github.com/elastic/elasticsearch/issues/31456 is fixed.
+  integTestRunner {
+    systemProperty 'tests.rest.blacklist', [
+            'ingest_attachment/20_attachment_processor/Test indexed chars are configurable',
+            'ingest_attachment/20_attachment_processor/Test indexed chars are configurable per document'
+    ].join(',')
+  }
+}
+
 dependencies {
   // mandatory for tika
   compile "org.apache.tika:tika-core:${versions.tika}"

plugins/ingest-attachment/licenses/bcmail-jdk15on-1.55.jar.sha1

@@ -0,0 +1 @@
+db389ade95f48592908a84e7050a691c8834723c

plugins/ingest-attachment/licenses/bcmail-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+9cef0aab8a4bb849a8476c058ce3ff302aba3fff