Update build to Gradle 7.0 (7.x back porting) (#68506) (#71613)

- Update gradle wrapper to gradle 7.0
- Remove deprecated usages to make build 7.0 compatible
- Fix excludes in docs snippet tasks (See gradle/gradle#16160 for details)
- Fix deprecation warnings in 7.0
- Add explicit dependencies that have been missed
- Make extract native licenses tasks output dir more explicit
- Use a snapshot of the ospackage plugin that includes a fix for 7.0 already
- fix test runtime classpath setup in repository-hdfs
- Make task dependency explicit to fix further deprecation warnings
- Remove manual check for http repo usages that has been deprecated in gradle 7.0
- Update spock to latest 2.0 milestone required for groovy 3

- Disable module inference in Eclipse (#71649)

We recently upgrade to gradle 7.0 (#69096) which turned on module
inference by default. I'm sure that's lovely, but its broken eclipse. We
should probably support modules one day, but that day ain't today. This
turns off module inference for eclipse so it can continue compiling
things just like it did yesterday.

Co-authored-by: Nik Everett <[email protected]>

Author: breskeby

build.gradle

@@ -283,7 +283,7 @@ allprojects {
         }
       }
       boolean hasShadow = project.plugins.hasPlugin(ShadowPlugin)
-      project.configurations.compile.dependencies
+      project.configurations.compileClasspath.dependencies
         .findAll()
         .toSorted(sortClosure)
         .each({ c -> depJavadocClosure(hasShadow, c) })
@@ -342,6 +342,7 @@ allprojects {
   }
 
   plugins.withType(JavaBasePlugin) {
+    java.modularity.inferModulePath.set(false)
     eclipse.classpath.file.whenMerged { classpath ->
       /*
        * give each source folder a unique corresponding output folder

buildSrc/build.gradle

@@ -117,7 +117,7 @@ dependencies {
   testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
   testImplementation 'org.mockito:mockito-core:1.9.5'
   testImplementation "org.hamcrest:hamcrest:${props.getProperty('hamcrest')}"
-  integTestImplementation('org.spockframework:spock-core:1.3-groovy-2.5') {
+  integTestImplementation('org.spockframework:spock-core:2.0-M5-groovy-3.0') {
     exclude module: "groovy"
   }
   integTestImplementation "org.xmlunit:xmlunit-core:2.8.2"

buildSrc/src/integTest/java/org/elasticsearch/gradle/BuildPluginIT.java

@@ -21,7 +21,6 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
 import java.util.List;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
@@ -45,36 +44,6 @@ public void testCheckTask() {
         assertTaskSuccessful(result, ":check");
     }
 
-    public void testInsecureMavenRepository() throws IOException {
-        final String name = "elastic-maven";
-        final String url = "http://s3.amazonaws.com/artifacts.elastic.co/maven";
-        // add an insecure maven repository to the build.gradle
-        final List<String> lines = Arrays.asList(
-            "repositories {",
-            "  maven {",
-            "    name \"elastic-maven\"",
-            "    url \"" + url + "\"\n",
-            "  }",
-            "}"
-        );
-        runInsecureArtifactRepositoryTest(name, url, lines);
-    }
-
-    public void testInsecureIvyRepository() throws IOException {
-        final String name = "elastic-ivy";
-        final String url = "http://s3.amazonaws.com/artifacts.elastic.co/ivy";
-        // add an insecure ivy repository to the build.gradle
-        final List<String> lines = Arrays.asList(
-            "repositories {",
-            "  ivy {",
-            "    name \"elastic-ivy\"",
-            "    url \"" + url + "\"\n",
-            "  }",
-            "}"
-        );
-        runInsecureArtifactRepositoryTest(name, url, lines);
-    }
-
     private void runInsecureArtifactRepositoryTest(final String name, final String url, final List<String> lines) throws IOException {
         final File projectDir = getProjectDir("elasticsearch.build");
         final Path projectDirPath = projectDir.toPath();

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -30,13 +30,13 @@ class SnippetsTask extends DefaultTask {
     private static final String SCHAR = /(?:\\\/|[^\/])/
     private static final String SUBSTITUTION = /s\/($SCHAR+)\/($SCHAR*)\//
     private static final String CATCH = /catch:\s*((?:\/[^\/]+\/)|[^ \]]+)/
-    private static final String SKIP = /skip:([^\]]+)/
+    private static final String SKIP_REGEX = /skip:([^\]]+)/
     private static final String SETUP = /setup:([^ \]]+)/
     private static final String TEARDOWN = /teardown:([^ \]]+)/
     private static final String WARNING = /warning:(.+)/
     private static final String NON_JSON = /(non_json)/
     private static final String TEST_SYNTAX =
-        /(?:$CATCH|$SUBSTITUTION|$SKIP|(continued)|$SETUP|$TEARDOWN|$WARNING|(skip_shard_failures)) ?/
+        /(?:$CATCH|$SUBSTITUTION|$SKIP_REGEX|(continued)|$SETUP|$TEARDOWN|$WARNING|(skip_shard_failures)) ?/
 
     /**
      * Action to take on each snippet. Called with a single parameter, an
@@ -51,12 +51,7 @@ class SnippetsTask extends DefaultTask {
      * directory.
      */
     @InputFiles
-    ConfigurableFileTree docs = project.fileTree(project.projectDir) {
-        // No snippets in the build file
-        exclude 'build.gradle'
-        // That is where the snippets go, not where they come from!
-        exclude 'build'
-    }
+    ConfigurableFileTree docs
 
     /**
      * Substitutions done on every snippet's contents.
@@ -251,7 +246,7 @@ class SnippetsTask extends DefaultTask {
                             substitutions = []
                         }
                         String loc = "$file:$lineNumber"
-                        parse(loc, matcher.group(2), /(?:$SUBSTITUTION|$NON_JSON|$SKIP) ?/) {
+                        parse(loc, matcher.group(2), /(?:$SUBSTITUTION|$NON_JSON|$SKIP_REGEX) ?/) {
                             if (it.group(1) != null) {
                                 // TESTRESPONSE[s/adsf/jkl/]
                                 substitutions.add([it.group(1), it.group(2)])

buildSrc/src/main/java/org/elasticsearch/gradle/RepositoriesSetupPlugin.java

@@ -12,7 +12,6 @@
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.dsl.RepositoryHandler;
-import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
 import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
 
 import java.net.MalformedURLException;
@@ -37,20 +36,6 @@ public void apply(Project project) {
      * Adds repositories used by ES projects and dependencies
      */
     public static void configureRepositories(Project project) {
-        // ensure all repositories use secure urls
-        // TODO: remove this with gradle 7.0, which no longer allows insecure urls
-        project.getRepositories().all(repository -> {
-            if (repository instanceof MavenArtifactRepository) {
-                final MavenArtifactRepository maven = (MavenArtifactRepository) repository;
-                assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl());
-                for (URI uri : maven.getArtifactUrls()) {
-                    assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri);
-                }
-            } else if (repository instanceof IvyArtifactRepository) {
-                final IvyArtifactRepository ivy = (IvyArtifactRepository) repository;
-                assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl());
-            }
-        });
         RepositoryHandler repos = project.getRepositories();
         if (System.getProperty("repos.mavenLocal") != null) {
             // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is

buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -24,7 +24,7 @@
 import org.gradle.internal.jvm.inspection.JvmMetadataDetector;
 import org.gradle.internal.jvm.inspection.JvmVendor;
 import org.gradle.jvm.toolchain.internal.InstallationLocation;
-import org.gradle.jvm.toolchain.internal.SharedJavaInstallationRegistry;
+import org.gradle.jvm.toolchain.internal.JavaInstallationRegistry;
 import org.gradle.util.GradleVersion;
 
 import javax.inject.Inject;
@@ -59,13 +59,13 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private static Integer _defaultParallel = null;
     private static Boolean _isBundledJdkSupported = null;
 
-    private final SharedJavaInstallationRegistry javaInstallationRegistry;
+    private final JavaInstallationRegistry javaInstallationRegistry;
     private final JvmMetadataDetector metadataDetector;
     private final ProviderFactory providers;
 
     @Inject
     public GlobalBuildInfoPlugin(
-        SharedJavaInstallationRegistry javaInstallationRegistry,
+        JavaInstallationRegistry javaInstallationRegistry,
         JvmMetadataDetector metadataDetector,
         ProviderFactory providers
     ) {

buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java

@@ -120,15 +120,15 @@ public class DependencyLicensesTask extends DefaultTask {
      * the LICENSE and NOTICE file for that jar.
      */
     public void mapping(Map<String, String> props) {
-        String from = props.remove("from");
+        String from = props.get("from");
         if (from == null) {
             throw new InvalidUserDataException("Missing \"from\" setting for license name mapping");
         }
-        String to = props.remove("to");
+        String to = props.get("to");
         if (to == null) {
             throw new InvalidUserDataException("Missing \"to\" setting for license name mapping");
         }
-        if (props.isEmpty() == false) {
+        if (props.size() > 2) {
             throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: " + props.keySet());
         }
         mappings.put(from, to);

buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsPrecommitPlugin.java

@@ -21,6 +21,7 @@
 
 public class FilePermissionsPrecommitPlugin extends PrecommitPlugin implements InternalPlugin {
 
+    public static final String FILEPERMISSIONS_TASK_NAME = "filepermissions";
     private ProviderFactory providerFactory;
 
     @Inject
@@ -32,7 +33,7 @@ public FilePermissionsPrecommitPlugin(ProviderFactory providerFactory) {
     public TaskProvider<? extends Task> createTask(Project project) {
         return project.getTasks()
             .register(
-                "filepermissions",
+                FILEPERMISSIONS_TASK_NAME,
                 FilePermissionsTask.class,
                 filePermissionsTask -> filePermissionsTask.getSources()
                     .addAll(

buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsPrecommitPlugin.java

@@ -21,6 +21,7 @@
 
 public class ForbiddenPatternsPrecommitPlugin extends PrecommitPlugin implements InternalPlugin {
 
+    public static final String FORBIDDEN_PATTERNS_TASK_NAME = "forbiddenPatterns";
     private final ProviderFactory providerFactory;
 
     @Inject
@@ -30,7 +31,7 @@ public ForbiddenPatternsPrecommitPlugin(ProviderFactory providerFactory) {
 
     @Override
     public TaskProvider<? extends Task> createTask(Project project) {
-        return project.getTasks().register("forbiddenPatterns", ForbiddenPatternsTask.class, forbiddenPatternsTask -> {
+        return project.getTasks().register(FORBIDDEN_PATTERNS_TASK_NAME, ForbiddenPatternsTask.class, forbiddenPatternsTask -> {
             forbiddenPatternsTask.getSourceFolders()
                 .addAll(
                     providerFactory.provider(

buildSrc/src/main/java/org/elasticsearch/gradle/test/TestWithSslPlugin.java

@@ -10,6 +10,8 @@
 
 import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask;
 import org.elasticsearch.gradle.info.BuildParams;
+import org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin;
+import org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin;
 import org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsTask;
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
 import org.elasticsearch.gradle.testclusters.TestClustersAware;
@@ -23,6 +25,9 @@
 
 import java.io.File;
 
+import static org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin.FILEPERMISSIONS_TASK_NAME;
+import static org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin.FORBIDDEN_PATTERNS_TASK_NAME;
+
 public class TestWithSslPlugin implements Plugin<Project> {
 
     @Override
@@ -38,14 +43,19 @@ public void apply(Project project) {
                 t.copy("test/ssl/test-node.jks");
                 t.setOutputDir(keyStoreDir);
             });
-
+        project.getPlugins()
+            .withType(ForbiddenPatternsPrecommitPlugin.class)
+            .configureEach(plugin -> project.getTasks().named(FORBIDDEN_PATTERNS_TASK_NAME).configure(t -> t.dependsOn(exportKeyStore)));
+        project.getPlugins()
+            .withType(FilePermissionsPrecommitPlugin.class)
+            .configureEach(
+                filePermissionPlugin -> project.getTasks().named(FILEPERMISSIONS_TASK_NAME).configure(t -> t.dependsOn(exportKeyStore))
+            );
         project.getPlugins().withType(StandaloneRestTestPlugin.class).configureEach(restTestPlugin -> {
             SourceSet testSourceSet = Util.getJavaTestSourceSet(project).get();
             testSourceSet.getResources().srcDir(new File(keyStoreDir, "test/ssl"));
-            testSourceSet.compiledBy(exportKeyStore);
-
+            project.getTasks().named(testSourceSet.getProcessResourcesTaskName()).configure(t -> t.dependsOn(exportKeyStore));
             project.getTasks().withType(TestClustersAware.class).configureEach(clusterAware -> clusterAware.dependsOn(exportKeyStore));
-
             // Tell the tests we're running with ssl enabled
             project.getTasks()
                 .withType(RestIntegTestTask.class)