Merge remote-tracking branch 'elastic/master' into deduplicate-index-metadata

Author: original-brownbear

.ci/bwcVersions

@@ -19,6 +19,7 @@ BWC_VERSION:
   - "7.6.2"
   - "7.7.0"
   - "7.7.1"
+  - "7.7.2"
   - "7.8.0"
   - "7.9.0"
   - "8.0.0"

CONTRIBUTING.md

@@ -115,10 +115,20 @@ onwards and Eclipse 2020-3 and onwards.
 
     ./gradlew :run
 
-You can access Elasticsearch with:
+That'll spend a while building Elasticsearch and then it'll start Elasticsearch,
+writing its log above Gradle's status message. We log a lot of stuff on startup,
+specifically these lines tell you that Elasticsearch is ready:
+
+    [2020-05-29T14:50:35,167][INFO ][o.e.h.AbstractHttpServerTransport] [runTask-0] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200}
+    [2020-05-29T14:50:35,169][INFO ][o.e.n.Node               ] [runTask-0] started
+
+But to be honest its typically easier to wait until the console stopps scrolling
+and then run `curl` in another window like this:
 
     curl -u elastic:password localhost:9200
 
+
+
 ### Importing the project into IntelliJ IDEA
 
 Elasticsearch builds using Java 14. When importing into IntelliJ you will need

build.gradle

@@ -46,6 +46,8 @@ apply from: 'gradle/ide.gradle'
 apply from: 'gradle/forbidden-dependencies.gradle'
 apply from: 'gradle/formatting.gradle'
 apply from: 'gradle/local-distribution.gradle'
+apply from: 'gradle/fips.gradle'
+apply from: 'gradle/run.gradle'
 
 // common maven publishing configuration
 allprojects {
@@ -374,43 +376,6 @@ allprojects {
   tasks.named('eclipse') { dependsOn 'cleanEclipse', 'copyEclipseSettings' }
 }
 
-// we need to add the same --debug-jvm option as
-// the real RunTask has, so we can pass it through
-class Run extends DefaultTask {
-  boolean debug = false
-
-  @Option(
-    option = "debug-jvm",
-    description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch."
-  )
-  public void setDebug(boolean enabled) {
-    project.project(':distribution').run.debug = enabled
-  }
-
-  @Option(
-    option = "data-dir",
-    description = "Override the base data directory used by the testcluster"
-  )
-  public void setDataDir(String dataDirStr) {
-    project.project(':distribution').run.dataDir = dataDirStr
-  }
-
-  @Option(
-    option = "keystore-password",
-    description = "Set the elasticsearch keystore password"
-  )
-  public void setKeystorePassword(String password) {
-    project.project(':distribution').run.keystorePassword = password
-  }
-}
-
-tasks.register("run", Run) {
-  dependsOn ':distribution:run'
-  description = 'Runs elasticsearch in the foreground'
-  group = 'Verification'
-  impliesSubProjects = true
-}
-
 wrapper {
   distributionType = 'ALL'
   doLast {

buildSrc/build.gradle

@@ -174,14 +174,15 @@ if (project != rootProject) {
     'forbiddenApisTest', 'forbiddenApisIntegTest', 'forbiddenApisTestFixtures')
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
-  if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))) {
+  if (org.elasticsearch.gradle.info.BuildParams.inFipsJvm) {
+    // We don't support running gradle with a JVM that is in FIPS 140 mode, so we don't test it.
+    // WaitForHttpResourceTests tests would fail as they use JKS/PKCS12 keystores
     test.enabled = false
+    testingConventions.enabled = false
   }
 
-  configurations {
-    distribution
-    reaper
-  }
+  configurations.register("distribution")
+  configurations.register("reaper")
 
   dependencies {
     reaper project('reaper')
@@ -221,6 +222,8 @@ if (project != rootProject) {
   forbiddenPatterns {
     exclude '**/*.wav'
     exclude '**/*.p12'
+    exclude '**/*.jks'
+    exclude '**/*.crt'
     // the file that actually defines nocommit
     exclude '**/ForbiddenPatternsTask.java'
     exclude '**/*.bcfks'

buildSrc/src/integTest/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTaskIT.java

@@ -32,36 +32,20 @@ public void testUpToDateWithSourcesConfigured() {
         BuildResult result = getGradleRunner(PROJECT_NAME).withArguments("buildResources", "-s", "-i").build();
         assertTaskSuccessful(result, ":buildResources");
         assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml");
-
-        // using task avoidance api means the task configuration of the sample task is never triggered
-        assertBuildFileDoesNotExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
 
         result = getGradleRunner(PROJECT_NAME).withArguments("buildResources", "-s", "-i").build();
         assertTaskUpToDate(result, ":buildResources");
         assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml");
-
-        // using task avoidance api means the task configuration of the sample task is never triggered
-        assertBuildFileDoesNotExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
-    }
-
-    public void testImplicitTaskDependencyCopy() {
-        BuildResult result = getGradleRunner(PROJECT_NAME).withArguments("clean", "sampleCopyAll", "-s", "-i").build();
-
-        assertTaskSuccessful(result, ":buildResources");
-        assertTaskSuccessful(result, ":sampleCopyAll");
-        assertBuildFileExists(result, PROJECT_NAME, "sampleCopyAll/checkstyle.xml");
-
-        // using task avoidance api means the task configuration of the sample task is never triggered
-        // which means buildResource is not configured to copy this file
-        assertBuildFileDoesNotExists(result, PROJECT_NAME, "sampleCopyAll/checkstyle_suppressions.xml");
+        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
     }
 
-    public void testImplicitTaskDependencyInputFileOfOther() {
-        BuildResult result = getGradleRunner(PROJECT_NAME).withArguments("clean", "sample", "-s", "-i").build();
+    public void testOutputAsInput() {
+        BuildResult result = getGradleRunner(PROJECT_NAME).withArguments("clean", "sampleCopy", "-s", "-i").build();
 
-        assertTaskSuccessful(result, ":sample");
-        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml");
-        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
+        assertTaskSuccessful(result, ":sampleCopy");
+        assertBuildFileExists(result, PROJECT_NAME, "sampleCopy/checkstyle.xml");
+        assertBuildFileExists(result, PROJECT_NAME, "sampleCopy/checkstyle_suppressions.xml");
     }
 
     public void testIncorrectUsage() {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -19,33 +19,15 @@
 package org.elasticsearch.gradle
 
 import groovy.transform.CompileStatic
-import org.apache.commons.io.IOUtils
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
 import org.elasticsearch.gradle.precommit.PrecommitTasks
-import org.elasticsearch.gradle.test.ErrorReportingTestListener
-import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
-import org.elasticsearch.gradle.testclusters.TestClustersPlugin
-import org.elasticsearch.gradle.util.GradleUtils
-import org.gradle.api.*
-import org.gradle.api.artifacts.Configuration
-import org.gradle.api.artifacts.Dependency
-import org.gradle.api.artifacts.ModuleDependency
-import org.gradle.api.artifacts.ProjectDependency
-import org.gradle.api.artifacts.dsl.RepositoryHandler
-import org.gradle.api.artifacts.repositories.ExclusiveContentRepository
-import org.gradle.api.artifacts.repositories.IvyArtifactRepository
-import org.gradle.api.artifacts.repositories.IvyPatternRepositoryLayout
-import org.gradle.api.artifacts.repositories.MavenArtifactRepository
-import org.gradle.api.credentials.HttpHeaderCredentials
-import org.gradle.api.execution.TaskActionListener
+import org.gradle.api.GradleException
+import org.gradle.api.InvalidUserDataException
+import org.gradle.api.Plugin
+import org.gradle.api.Project
 import org.gradle.api.file.CopySpec
 import org.gradle.api.plugins.ExtraPropertiesExtension
-import org.gradle.api.plugins.JavaPlugin
 import org.gradle.api.tasks.bundling.Jar
-import org.gradle.api.tasks.testing.Test
-import org.gradle.util.GradleVersion
-
-import java.nio.charset.StandardCharsets
 
 /**
  * Encapsulates build configuration for elasticsearch projects.
@@ -68,59 +50,8 @@ class BuildPlugin implements Plugin<Project> {
         project.pluginManager.apply('elasticsearch.publish')
         project.pluginManager.apply(DependenciesInfoPlugin)
 
-        project.getTasks().register("buildResources", ExportElasticsearchBuildResourcesTask)
-
         project.extensions.getByType(ExtraPropertiesExtension).set('versions', VersionProperties.versions)
         PrecommitTasks.create(project, true)
-        configureFips140(project)
-    }
-
-    static void configureFips140(Project project) {
-        // Common config when running with a FIPS-140 runtime JVM
-        if (inFipsJvm()) {
-            ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources') as ExportElasticsearchBuildResourcesTask
-            File securityProperties = buildResources.copy("fips_java.security")
-            File securityPolicy = buildResources.copy("fips_java.policy")
-            File bcfksKeystore = buildResources.copy("cacerts.bcfks")
-            // This configuration can be removed once system modules are available
-            GradleUtils.maybeCreate(project.configurations, 'extraJars') {
-                project.dependencies.add('extraJars', "org.bouncycastle:bc-fips:1.0.1")
-                project.dependencies.add('extraJars', "org.bouncycastle:bctls-fips:1.0.9")
-            }
-            project.pluginManager.withPlugin("elasticsearch.testclusters") {
-                NamedDomainObjectContainer<ElasticsearchCluster> testClusters = project.extensions.findByName(TestClustersPlugin.EXTENSION_NAME) as NamedDomainObjectContainer<ElasticsearchCluster>
-                testClusters.all { ElasticsearchCluster cluster ->
-                    for (File dep : project.getConfigurations().getByName("extraJars").getFiles()){
-                        cluster.extraJarFile(dep)
-                    }
-                    cluster.extraConfigFile("fips_java.security", securityProperties)
-                    cluster.extraConfigFile("fips_java.policy", securityPolicy)
-                    cluster.extraConfigFile("cacerts.bcfks", bcfksKeystore)
-                    cluster.systemProperty('java.security.properties', '=${ES_PATH_CONF}/fips_java.security')
-                    cluster.systemProperty('java.security.policy', '=${ES_PATH_CONF}/fips_java.policy')
-                    cluster.systemProperty('javax.net.ssl.trustStore', '${ES_PATH_CONF}/cacerts.bcfks')
-                    cluster.systemProperty('javax.net.ssl.trustStorePassword', 'password')
-                    cluster.systemProperty('javax.net.ssl.keyStorePassword', 'password')
-                    cluster.systemProperty('javax.net.ssl.keyStoreType', 'BCFKS')
-                }
-            }
-            project.tasks.withType(Test).configureEach { Test task ->
-                task.dependsOn(buildResources)
-                task.systemProperty('javax.net.ssl.trustStorePassword', 'password')
-                task.systemProperty('javax.net.ssl.keyStorePassword', 'password')
-                task.systemProperty('javax.net.ssl.trustStoreType', 'BCFKS')
-                // Using the key==value format to override default JVM security settings and policy
-                // see also: https://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html
-                task.systemProperty('java.security.properties', String.format(Locale.ROOT, "=%s", securityProperties.toString()))
-                task.systemProperty('java.security.policy', String.format(Locale.ROOT, "=%s", securityPolicy.toString()))
-                task.systemProperty('javax.net.ssl.trustStore', bcfksKeystore.toString())
-            }
-
-        }
-    }
-
-    private static inFipsJvm(){
-        return Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
     }
 
     static void configureLicenseAndNotice(Project project) {