Create gradle plugin for ES stable plugins (#90355)

New ES stable plugins when built should have a stable-plugin-descriptor.properties file instead of plugin-descriptor.properties.
New plugins also do not use classname property in the plugin descriptor
new plugin will also scan classes and libraries for @NamedComponents and will create named_components.json file. That file contains a map of Extensible interface (like TokenizerFactory) to a map of "component name" to "className"

This commit extracts common logic from PluginBuildPlugin into BasePluginBuildPLugin so that it can also be used by StableBuildPlugin
the differences are:
classaname - used in old plugin, but not in the new one (stable)
the plugin descriptor file name - the new one has stable-plugin-descriptor.properties
dependencies - the new plugin does not need elasticsearch as a dependency.
We might want to consider if we want to add test framework dependency in the future.

relates #88980

Author: pgomulka

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/ElasticsearchDistributionExtension.java

@@ -19,7 +19,7 @@
 import java.util.concurrent.Callable;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.gradle.plugin.PluginBuildPlugin.EXPLODED_BUNDLE_CONFIG;
+import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.EXPLODED_BUNDLE_CONFIG;
 
 public class ElasticsearchDistributionExtension {
     private static final Pattern CONFIG_BIN_REGEX_PATTERN = Pattern.compile("([^\\/]+\\/)?(config|bin)\\/.*");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestTestBasePlugin.java

@@ -28,8 +28,8 @@
 
 import javax.inject.Inject;
 
-import static org.elasticsearch.gradle.plugin.PluginBuildPlugin.BUNDLE_PLUGIN_TASK_NAME;
-import static org.elasticsearch.gradle.plugin.PluginBuildPlugin.EXPLODED_BUNDLE_PLUGIN_TASK_NAME;
+import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.BUNDLE_PLUGIN_TASK_NAME;
+import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.EXPLODED_BUNDLE_PLUGIN_TASK_NAME;
 
 public class RestTestBasePlugin implements Plugin<Project> {
     private static final String TESTS_REST_CLUSTER = "tests.rest.cluster";

build-tools/build.gradle

@@ -37,6 +37,10 @@ gradlePlugin {
             id = 'elasticsearch.esplugin'
             implementationClass = 'org.elasticsearch.gradle.plugin.PluginBuildPlugin'
         }
+        stableEsPlugin {
+          id = 'elasticsearch.stable-esplugin'
+          implementationClass = 'org.elasticsearch.gradle.plugin.StablePluginBuildPlugin'
+        }
         javaRestTest {
             id = 'elasticsearch.java-rest-test'
             implementationClass = 'org.elasticsearch.gradle.test.JavaRestTestPlugin'
@@ -119,6 +123,8 @@ dependencies {
     api buildLibs.commmons.io
     implementation buildLibs.asm.tree
     implementation buildLibs.asm
+    implementation buildLibs.jackson.core
+    implementation buildLibs.jackson.databind
 
     testFixturesApi gradleApi()
     testFixturesApi gradleTestKit()
@@ -128,7 +134,9 @@ dependencies {
     testFixturesApi(buildLibs.spock.core) {
         exclude module: "groovy"
     }
-
+    testFixturesApi(buildLibs.bytebuddy) {
+      because 'Generating dynamic plugin apis'
+    }
     testImplementation(buildLibs.spock.junit4) {
         because 'required as we rely on junit4 rules'
     }
@@ -144,6 +152,7 @@ dependencies {
     testRuntimeOnly(buildLibs.junit5.platform.launcher) {
         because 'allows tests to run from IDEs that bundle older version of launcher'
     }
+
 }
 
 tasks.named('test').configure {

build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy

@@ -75,7 +75,7 @@ class PluginBuildPluginFuncTest extends AbstractGradleFuncTest {
             }
 
             dependencies {
-                consume project(path:':', configuration:'${PluginBuildPlugin.EXPLODED_BUNDLE_CONFIG}')
+                consume project(path:':', configuration:'${BasePluginBuildPlugin.EXPLODED_BUNDLE_CONFIG}')
             }
 
             tasks.register("resolveModule", Copy) {

build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StableBuildPluginPluginFuncTest.groovy

@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.plugin
+
+import groovy.json.JsonSlurper
+
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.elasticsearch.gradle.internal.test.StableApiJarMocks
+import org.gradle.testkit.runner.TaskOutcome
+
+import java.nio.file.Files
+import java.nio.file.Path
+import java.util.stream.Collectors
+
+class StableBuildPluginPluginFuncTest extends AbstractGradleFuncTest {
+
+    def setup() {
+        // underlaying TestClusterPlugin and StandaloneRestIntegTestTask are not cc compatible
+        configurationCacheCompatible = false
+    }
+
+    def "can build stable plugin properties"() {
+        given:
+        buildFile << """plugins {
+                id 'elasticsearch.stable-esplugin'
+            }
+
+            version = '1.2.3'
+
+            esplugin {
+                name = 'myplugin'
+                description = 'test plugin'
+            }
+            """
+
+        when:
+        def result = gradleRunner(":pluginProperties").build()
+        def props = getPluginProperties()
+
+        then:
+        result.task(":pluginProperties").outcome == TaskOutcome.SUCCESS
+        props.get("classname") == null
+
+        props.get("name") == "myplugin"
+        props.get("version") == "1.2.3"
+        props.get("description") == "test plugin"
+        props.get("modulename") == ""
+        props.get("java.version") == Integer.toString(Runtime.version().feature())
+        props.get("elasticsearch.version") == VersionProperties.elasticsearchVersion.toString()
+        props.get("extended.plugins") == ""
+        props.get("has.native.controller") == "false"
+        props.size() == 8
+
+    }
+
+    def "can scan and create named components file"() {
+        given:
+        File jarFolder = new File(testProjectDir.root, "jars")
+        jarFolder.mkdirs()
+
+        buildFile << """plugins {
+                id 'elasticsearch.stable-esplugin'
+            }
+
+            version = '1.2.3'
+
+            esplugin {
+                name = 'myplugin'
+                description = 'test plugin'
+            }
+
+            dependencies {
+                implementation files('${StableApiJarMocks.createPluginApiJar(jarFolder.toPath()).toAbsolutePath()}')
+                implementation files('${StableApiJarMocks.createExtensibleApiJar(jarFolder.toPath()).toAbsolutePath()}')
+            }
+
+            """
+
+        file("src/main/java/org/acme/A.java") << """
+            package org.acme;
+
+            import org.elasticsearch.plugin.api.NamedComponent;
+            import org.elasticsearch.plugin.scanner.test_classes.ExtensibleClass;
+
+            @NamedComponent(name = "componentA")
+            public class A extends ExtensibleClass {
+            }
+        """
+
+
+        when:
+        def result = gradleRunner(":assemble").build()
+        Path namedComponents = file("build/generated-named-components/named_components.json").toPath();
+        def map = new JsonSlurper().parse(namedComponents.toFile())
+        then:
+        result.task(":assemble").outcome == TaskOutcome.SUCCESS
+
+        map  == ["org.elasticsearch.plugin.scanner.test_classes.ExtensibleClass" : (["componentA" : "org.acme.A"]) ]
+    }
+
+
+    Map<String, String> getPluginProperties() {
+        Path propsFile = file("build/generated-descriptor/stable-plugin-descriptor.properties").toPath();
+        Properties rawProps = new Properties()
+        try (var inputStream = Files.newInputStream(propsFile)) {
+            rawProps.load(inputStream)
+        }
+        return rawProps.entrySet().stream().collect(Collectors.toMap(e -> e.getKey().toString(), e -> e.getValue().toString()))
+    }
+}

build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java

@@ -0,0 +1,187 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.plugin;
+
+import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
+import org.elasticsearch.gradle.jarhell.JarHellPlugin;
+import org.elasticsearch.gradle.test.GradleTestPolicySetupPlugin;
+import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
+import org.elasticsearch.gradle.testclusters.RunTask;
+import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.Transformer;
+import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+import org.gradle.api.file.CopySpec;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.file.RegularFile;
+import org.gradle.api.plugins.BasePlugin;
+import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.plugins.JavaPluginExtension;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.SourceSetContainer;
+import org.gradle.api.tasks.Sync;
+import org.gradle.api.tasks.TaskProvider;
+import org.gradle.api.tasks.bundling.Zip;
+
+import java.io.File;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import javax.inject.Inject;
+
+/**
+ * Common logic for building ES plugins.
+ * Requires plugin extension to be created before applying
+ */
+public class BasePluginBuildPlugin implements Plugin<Project> {
+
+    public static final String PLUGIN_EXTENSION_NAME = "esplugin";
+    public static final String BUNDLE_PLUGIN_TASK_NAME = "bundlePlugin";
+    public static final String EXPLODED_BUNDLE_PLUGIN_TASK_NAME = "explodedBundlePlugin";
+    public static final String EXPLODED_BUNDLE_CONFIG = "explodedBundleZip";
+
+    protected final ProviderFactory providerFactory;
+
+    @Inject
+    public BasePluginBuildPlugin(ProviderFactory providerFactory) {
+        this.providerFactory = providerFactory;
+    }
+
+    @Override
+    public void apply(final Project project) {
+        project.getPluginManager().apply(JavaPlugin.class);
+        project.getPluginManager().apply(TestClustersPlugin.class);
+        project.getPluginManager().apply(CompileOnlyResolvePlugin.class);
+        project.getPluginManager().apply(JarHellPlugin.class);
+        project.getPluginManager().apply(GradleTestPolicySetupPlugin.class);
+
+        var extension = project.getExtensions()
+            .create(BasePluginBuildPlugin.PLUGIN_EXTENSION_NAME, PluginPropertiesExtension.class, project);
+
+        final var bundleTask = createBundleTasks(project, extension);
+        project.getConfigurations().getByName("default").extendsFrom(project.getConfigurations().getByName("runtimeClasspath"));
+
+        // allow running ES with this plugin in the foreground of a build
+        var testClusters = testClusters(project, TestClustersPlugin.EXTENSION_NAME);
+        var runCluster = testClusters.register("runTask", c -> {
+            // TODO: use explodedPlugin here for modules
+            if (GradleUtils.isModuleProject(project.getPath())) {
+                c.module(bundleTask.flatMap((Transformer<Provider<RegularFile>, Zip>) zip -> zip.getArchiveFile()));
+            } else {
+                c.plugin(bundleTask.flatMap((Transformer<Provider<RegularFile>, Zip>) zip -> zip.getArchiveFile()));
+            }
+        });
+
+        project.getTasks().register("run", RunTask.class, r -> {
+            r.useCluster(runCluster);
+            r.dependsOn(project.getTasks().named(BUNDLE_PLUGIN_TASK_NAME));
+        });
+    }
+
+    @SuppressWarnings("unchecked")
+    private static NamedDomainObjectContainer<ElasticsearchCluster> testClusters(Project project, String extensionName) {
+        return (NamedDomainObjectContainer<ElasticsearchCluster>) project.getExtensions().getByName(extensionName);
+    }
+
+    /**
+     * Adds bundle tasks which builds the dir and zip containing the plugin jars,
+     * metadata, properties, and packaging files
+     */
+    private TaskProvider<Zip> createBundleTasks(final Project project, PluginPropertiesExtension extension) {
+        final var pluginMetadata = project.file("src/main/plugin-metadata");
+
+        final var buildProperties = project.getTasks().register("pluginProperties", GeneratePluginPropertiesTask.class, task -> {
+            task.getPluginName().set(providerFactory.provider(extension::getName));
+            task.getPluginDescription().set(providerFactory.provider(extension::getDescription));
+            task.getPluginVersion().set(providerFactory.provider(extension::getVersion));
+            task.getElasticsearchVersion().set(Version.fromString(VersionProperties.getElasticsearch()).toString());
+            var javaExtension = project.getExtensions().getByType(JavaPluginExtension.class);
+            task.getJavaVersion().set(providerFactory.provider(() -> javaExtension.getTargetCompatibility().toString()));
+            task.getExtendedPlugins().set(providerFactory.provider(extension::getExtendedPlugins));
+            task.getHasNativeController().set(providerFactory.provider(extension::isHasNativeController));
+            task.getRequiresKeystore().set(providerFactory.provider(extension::isRequiresKeystore));
+            task.getIsLicensed().set(providerFactory.provider(extension::isLicensed));
+
+            var mainSourceSet = project.getExtensions().getByType(SourceSetContainer.class).getByName(SourceSet.MAIN_SOURCE_SET_NAME);
+            FileCollection moduleInfoFile = mainSourceSet.getOutput().getAsFileTree().matching(p -> p.include("module-info.class"));
+            task.getModuleInfoFile().setFrom(moduleInfoFile);
+
+        });
+        // add the plugin properties and metadata to test resources, so unit tests can
+        // know about the plugin (used by test security code to statically initialize the plugin in unit tests)
+        var testSourceSet = project.getExtensions().getByType(SourceSetContainer.class).getByName("test");
+        Map<String, Object> map = Map.of("builtBy", buildProperties);
+        testSourceSet.getOutput().dir(map, new File(project.getBuildDir(), "generated-resources"));
+        testSourceSet.getResources().srcDir(pluginMetadata);
+
+        var bundleSpec = createBundleSpec(project, pluginMetadata, buildProperties);
+        extension.setBundleSpec(bundleSpec);
+        // create the actual bundle task, which zips up all the files for the plugin
+        final var bundle = project.getTasks().register("bundlePlugin", Zip.class, zip -> zip.with(bundleSpec));
+        project.getTasks().named(BasePlugin.ASSEMBLE_TASK_NAME).configure(task -> task.dependsOn(bundle));
+
+        // also make the zip available as a configuration (used when depending on this project)
+        var configuration = project.getConfigurations().create("zip");
+        configuration.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE);
+        project.getArtifacts().add("zip", bundle);
+
+        var explodedBundle = project.getTasks().register(EXPLODED_BUNDLE_PLUGIN_TASK_NAME, Sync.class, sync -> {
+            sync.with(bundleSpec);
+            sync.into(new File(project.getBuildDir(), "explodedBundle/" + extension.getName()));
+        });
+
+        // also make the exploded bundle available as a configuration (used when depending on this project)
+        var explodedBundleZip = project.getConfigurations().create(EXPLODED_BUNDLE_CONFIG);
+        explodedBundleZip.setCanBeResolved(false);
+        explodedBundleZip.setCanBeConsumed(true);
+        explodedBundleZip.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
+        project.getArtifacts().add(EXPLODED_BUNDLE_CONFIG, explodedBundle);
+        return bundle;
+    }
+
+    private static CopySpec createBundleSpec(
+        Project project,
+        File pluginMetadata,
+        TaskProvider<GeneratePluginPropertiesTask> buildProperties
+    ) {
+        var bundleSpec = project.copySpec();
+        bundleSpec.from(buildProperties);
+        bundleSpec.from(pluginMetadata, copySpec -> {
+            // metadata (eg custom security policy)
+            // the codebases properties file is only for tests and not needed in production
+            copySpec.exclude("plugin-security.codebases");
+        });
+        bundleSpec.from(
+            (Callable<TaskProvider<Task>>) () -> project.getPluginManager().hasPlugin("com.github.johnrengelman.shadow")
+                ? project.getTasks().named("shadowJar")
+                : project.getTasks().named("jar")
+        );
+        bundleSpec.from(
+            project.getConfigurations()
+                .getByName("runtimeClasspath")
+                .minus(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME))
+        );
+
+        // extra files for the plugin to go into the zip
+        bundleSpec.from("src/main/packaging");// TODO: move all config/bin/_size/etc into packaging
+        bundleSpec.from("src/main", copySpec -> {
+            copySpec.include("config/**");
+            copySpec.include("bin/**");
+        });
+        return bundleSpec;
+    }
+}