From bad4ec7355b3bcb2bbd615b69eeabe224d8d812b Mon Sep 17 00:00:00 2001 From: Trevor Crawford Date: Wed, 15 Dec 2021 12:54:39 -0500 Subject: [PATCH] Generate features (#674) * Add Generate Features prototype and implement the necessary interfaces. Signed-off-by: Paul Gooderham * Add the generateFeatures command line and build file option. Signed-off-by: Paul Gooderham * Add comment to existing server.xml and scan gradle classes dirs. Signed-off-by: Paul Gooderham * Code cleanup. Signed-off-by: Paul Gooderham * Make generateFeatures option command line only. Clean up streams code. Signed-off-by: Paul Gooderham * Allow generateFeatures=false * Add classFile option to GenerateFeaturesTask (#640) Signed-off-by: Kathryn Kodama * Get binary scanner jar from connected repos * Fix name of API method. Signed-off-by: Paul Gooderham * Use gradle convention for latest release version * Update message * Change generated file name to generated-features.xml (#644) Signed-off-by: Kathryn Kodama * Fix NPE and remove getAllServerFeatures * Fix IO error when server is not yet setup. Signed-off-by: Paul Gooderham * Quick fix Signed-off-by: Paul Gooderham * Remove binary scanner jar parameter * Handle optimize and non-optimize cases * Add a system where the binary scanner handler detects an error generated by the scanner and calls it again with fewer restrictions in order to generate a list of suggested features that should work. This error and suggested solution is then displayed to the user. * Improved error reporting after code review. Signed-off-by: Paul Gooderham * After writing server.xml ensure element is on a new line. Signed-off-by: Paul Gooderham * Move the xml document rewrite code into ci.common. Signed-off-by: Paul Gooderham * Remove unused import. Signed-off-by: Paul Gooderham * Use the binary scanner support in ci.common. Signed-off-by: Paul Gooderham * Remove unused imports and class variable. Signed-off-by: Paul Gooderham * Allow info messages to appear in output by default and without setting log level to Info. Signed-off-by: Paul Gooderham * Get existing features for feature gen from the source config dir (#661) * Get existing features for feature gen from the source config dir Signed-off-by: Kathryn Kodama * Call generate features on dev mode startup and restart Signed-off-by: Kathryn Kodama * Simplify getServerFeatureUtil method Signed-off-by: Kathryn Kodama * Update to use latest snapshot from ci.common Signed-off-by: Kathryn Kodama * Add binary scanner sonatype repo to dev test Signed-off-by: Kathryn Kodama * Update sonatype repo link in dev tests Signed-off-by: Kathryn Kodama * Diable feature generation in dev tests Signed-off-by: Kathryn Kodama * Added optimize parameter for generating features * Added optimize parameter to generateFeatures task * Feature generation, create generated xml file with empty feature list (#664) * When no features left to generate create generated xml file with empty feature list Signed-off-by: Kathryn Kodama * Improve comment in generated-features.xml Signed-off-by: Kathryn Kodama * Improve formatting for comments Signed-off-by: Kathryn Kodama * Allow optimize to be passed in as a String parameter * Use binary scanner Maven coordinates promoted by the scanner team. Signed-off-by: Paul Gooderham * Fix compileJava call in recompileBuildFile * Put snapshot repo on project level for dev tests Signed-off-by: Kathryn Kodama * Updated debug statements * Use new method downloadBuildArtifact() to search buildscript portion of build.gradle Signed-off-by: Paul Gooderham * Exception handling for generateFeatures (#672) * Exception handling for generateFeatures Signed-off-by: Kathryn Kodama * Updated generateFeatures error message and generated-features.xml comment Signed-off-by: Kathryn Kodama * Remove unnecessary imports and adjust copyright header Signed-off-by: Kathryn Kodama * Update dev test project build.gradle Signed-off-by: Kathryn Kodama * Improve comment for default generateFeatures behaviour Signed-off-by: Kathryn Kodama * Improve exception handling message when generate features fails (#675) * Improve exception handling message when generate features fails Signed-off-by: Kathryn Kodama * Add return boolean for libertyGenerateFeatures method Signed-off-by: Kathryn Kodama * Add missing return statement for createNewInstallFeatureUtil (#678) Signed-off-by: Kathryn Kodama Co-authored-by: Paul Gooderham Co-authored-by: Eric Lau Co-authored-by: Kathryn Kodama --- .../tools/gradle/LibertyTaskFactory.groovy | 4 +- .../gradle/tasks/AbstractFeatureTask.groovy | 225 ++++++----- .../gradle/tasks/AbstractServerTask.groovy | 4 +- .../tools/gradle/tasks/DevTask.groovy | 136 ++++--- .../gradle/tasks/GenerateFeaturesTask.groovy | 348 ++++++++++++++++++ .../gradle/utils/ArtifactDownloadUtil.groovy | 57 +++ .../openliberty/tools/gradle/DevTest.groovy | 3 +- .../dev-test/basic-dev-project/build.gradle | 4 + 8 files changed, 625 insertions(+), 156 deletions(-) create mode 100644 src/main/groovy/io/openliberty/tools/gradle/tasks/GenerateFeaturesTask.groovy create mode 100644 src/main/groovy/io/openliberty/tools/gradle/utils/ArtifactDownloadUtil.groovy diff --git a/src/main/groovy/io/openliberty/tools/gradle/LibertyTaskFactory.groovy b/src/main/groovy/io/openliberty/tools/gradle/LibertyTaskFactory.groovy index 22eed0551..77f8f934f 100644 --- a/src/main/groovy/io/openliberty/tools/gradle/LibertyTaskFactory.groovy +++ b/src/main/groovy/io/openliberty/tools/gradle/LibertyTaskFactory.groovy @@ -30,6 +30,7 @@ import io.openliberty.tools.gradle.tasks.DebugTask import io.openliberty.tools.gradle.tasks.DeployTask import io.openliberty.tools.gradle.tasks.UndeployTask import io.openliberty.tools.gradle.tasks.InstallFeatureTask +import io.openliberty.tools.gradle.tasks.GenerateFeaturesTask import io.openliberty.tools.gradle.tasks.PrepareFeatureTask import io.openliberty.tools.gradle.tasks.InstallLibertyTask import io.openliberty.tools.gradle.tasks.UninstallFeatureTask @@ -62,7 +63,8 @@ class LibertyTaskFactory { project.tasks.create('deploy', DeployTask) project.tasks.create('undeploy', UndeployTask) project.tasks.create('installFeature', InstallFeatureTask) - project.tasks.create('prepareFeature', PrepareFeatureTask) + project.tasks.create('generateFeatures', GenerateFeaturesTask) + project.tasks.create('prepareFeature', PrepareFeatureTask) project.tasks.create('uninstallFeature', UninstallFeatureTask) project.tasks.create('cleanDirs', CleanTask) project.tasks.create('configureArquillian', ConfigureArquillianTask) diff --git a/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractFeatureTask.groovy b/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractFeatureTask.groovy index 2e37eafd2..e2b0a2d75 100644 --- a/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractFeatureTask.groovy +++ b/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractFeatureTask.groovy @@ -1,74 +1,104 @@ /** -* (C) Copyright IBM Corporation 2021. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * (C) Copyright IBM Corporation 2021. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.openliberty.tools.gradle.tasks -import java.util.Set - -import org.gradle.api.artifacts.ResolveException -import org.gradle.api.logging.LogLevel -import org.gradle.api.tasks.Internal -import org.gradle.api.tasks.TaskAction -import org.gradle.api.tasks.options.Option -import org.gradle.api.tasks.Input -import org.gradle.api.Project -import org.gradle.testfixtures.ProjectBuilder import io.openliberty.tools.common.plugins.util.InstallFeatureUtil import io.openliberty.tools.common.plugins.util.InstallFeatureUtil.ProductProperties import io.openliberty.tools.common.plugins.util.PluginExecutionException import io.openliberty.tools.common.plugins.util.PluginScenarioException +import io.openliberty.tools.common.plugins.util.ServerFeatureUtil +import io.openliberty.tools.gradle.utils.ArtifactDownloadUtil +import org.gradle.api.Project +import org.gradle.api.logging.LogLevel +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.options.Option +import org.gradle.testfixtures.ProjectBuilder public class AbstractFeatureTask extends AbstractServerTask { // DevMode uses this option to provide the location of the // temporary serverDir it uses after a change to the server.xml private String serverDirectoryParam; - + public boolean installFeaturesFromAnt; private InstallFeatureUtil util; - + @Internal - Project newProject = project; + Project newProject = project; + + private ServerFeatureUtil servUtil; @Option(option = 'serverDir', description = '(Optional) Server directory to get the list of features from.') void setServerDirectoryParam(String serverDir) { this.serverDirectoryParam = serverDir; } + private class ServerFeatureTaskUtil extends ServerFeatureUtil { + + @Override + void debug(String msg) { + logger.debug(msg); + } + + @Override + void error(String msg, Throwable throwable) { + logger.error(msg, e); + } + + @Override + void debug(String msg, Throwable throwable) { + logger.debug(msg, (Throwable) e); + } + + @Override + void debug(Throwable throwable) { + logger.debug("Throwable exception received: " + e.getMessage(), (Throwable) e); + } + + @Override + void warn(String msg) { + logger.warn(msg); + } + + @Override + void info(String msg) { + logger.lifecycle(msg); + } + } + private class InstallFeatureTaskUtil extends InstallFeatureUtil { - - public InstallFeatureTaskUtil(File installDir, String from, String to, Set pluginListedEsas, List propertiesList, String openLibertyVerion, String containerName, List additionalJsons) throws PluginScenarioException, PluginExecutionException { super(installDir, from, to, pluginListedEsas, propertiesList, openLibertyVerion, containerName, additionalJsons) } @Override public void debug(String msg) { - logger.debug(msg) + logger.debug(msg) } @Override public void debug(String msg, Throwable e) { - logger.debug(msg, (Throwable) e) + logger.debug(msg, (Throwable) e) } @Override public void debug(Throwable e) { - logger.debug("Throwable exception received: "+e.getMessage(), (Throwable) e) + logger.debug("Throwable exception received: " + e.getMessage(), (Throwable) e) } @Override @@ -98,28 +128,7 @@ public class AbstractFeatureTask extends AbstractServerTask { @Override public File downloadArtifact(String groupId, String artifactId, String type, String version) throws PluginExecutionException { - - String coordinates = groupId + ":" + artifactId + ":" + version + "@" + type - - def dep = newProject.dependencies.create(coordinates) - def config = newProject.configurations.detachedConfiguration(dep) - - Set files = new HashSet() - try { - config.resolvedConfiguration.resolvedArtifacts.each { artifact -> - File artifactFile = artifact.file - files.add(artifactFile) - debug(artifactFile.toString()) - } - } catch (ResolveException e) { - throw new PluginExecutionException("Could not find artifact with coordinates " + coordinates, e) - } - - if (!files) { - throw new PluginExecutionException("Could not find artifact with coordinates " + coordinates) - } - return files.iterator().next() - + return ArtifactDownloadUtil.downloadArtifact(project, groupId, artifactId, type, version); } } @@ -145,43 +154,32 @@ public class AbstractFeatureTask extends AbstractServerTask { } return features } - - @Internal - protected List getAdditionalJsonList() { - List result = new ArrayList() - project.configurations.featuresBom.dependencies.each { dep -> - def coordinate = dep.group + ":" + "features" + ":" + dep.version - logger.debug("feature Json: " + coordinate) - result.add(coordinate) - } - return result; - } - protected Set getSpecifiedFeatures(String containerName) throws PluginExecutionException { - if (util == null) { - def pluginListedEsas = getPluginListedFeatures(true) - def propertiesList = null; - def openLibertyVersion = null; - if (containerName == null) { - propertiesList = InstallFeatureUtil.loadProperties(getInstallDir(project)) - openLibertyVersion = InstallFeatureUtil.getOpenLibertyVersion(propertiesList) - } - def additionalJsons = getAdditionalJsonList() - createNewInstallFeatureUtil(pluginListedEsas, propertiesList, openLibertyVersion, containerName, additionalJsons) + @Internal + protected List getAdditionalJsonList() { + List result = new ArrayList() + project.configurations.featuresBom.dependencies.each { dep -> + def coordinate = dep.group + ":" + "features" + ":" + dep.version + logger.debug("feature Json: " + coordinate) + result.add(coordinate) } + return result; + } + + protected Set getSpecifiedFeatures(String containerName) throws PluginExecutionException { + InstallFeatureUtil util = getInstallFeatureUtil(null, containerName); // if createNewInstallFeatureUtil failed to create a new InstallFeatureUtil instance, then features are installed via ant - if(installFeaturesFromAnt) { + if (installFeaturesFromAnt) { Set featuresInstalledFromAnt; - if(server.features.name != null) { + if (server.features.name != null) { featuresInstalledFromAnt = new HashSet(server.features.name); return featuresInstalledFromAnt; - } - else { + } else { featuresInstalledFromAnt = new HashSet(); return featuresInstalledFromAnt; } } - + def pluginListedFeatures = getPluginListedFeatures(false) def dependencyFeatures = getDependencyFeatures() def serverFeatures = null; @@ -194,40 +192,65 @@ public class AbstractFeatureTask extends AbstractServerTask { } Set featuresToInstall = InstallFeatureUtil.combineToSet(pluginListedFeatures, dependencyFeatures, serverFeatures) - return featuresToInstall + return featuresToInstall + } + + protected ServerFeatureUtil getServerFeatureUtil() { + if (servUtil == null) { + servUtil = new ServerFeatureTaskUtil(); + } + return servUtil; } private void createNewInstallFeatureUtil(Set pluginListedEsas, List propertiesList, String openLibertyVerion, String containerName, List additionalJsons) throws PluginExecutionException { try { util = new InstallFeatureTaskUtil(getInstallDir(project), server.features.from, server.features.to, pluginListedEsas, propertiesList, openLibertyVerion, containerName, additionalJsons) } catch (PluginScenarioException e) { - logger.debug("Exception received: "+e.getMessage(),(Throwable)e) + logger.debug("Exception received: " + e.getMessage(), (Throwable) e) logger.debug("Installing features from installUtility.") installFeaturesFromAnt = true return } } + protected InstallFeatureUtil getInstallFeatureUtil(Set pluginListedEsas, String containerName) throws PluginExecutionException { + if (util == null) { + if (pluginListedEsas == null) { + pluginListedEsas = getPluginListedFeatures(true); + } + def propertiesList = null; + def openLibertyVersion = null; + if (containerName == null) { + propertiesList = InstallFeatureUtil.loadProperties(getInstallDir(project)) + openLibertyVersion = InstallFeatureUtil.getOpenLibertyVersion(propertiesList) + } + def additionalJsons = getAdditionalJsonList() + createNewInstallFeatureUtil(pluginListedEsas, propertiesList, openLibertyVersion, containerName, additionalJsons) + } + return util; + } + protected InstallFeatureUtil getInstallFeatureUtil(Set pluginListedEsas, List propertiesList, String openLibertyVerion, String containerName, List additionalJsons) throws PluginExecutionException { - //if installing userFeature, recompile gradle to find mavenLocal artifacts created by prepareFeature task. - if(project.configurations.featuresBom.dependencies) { - try { - ProjectBuilder builder = ProjectBuilder.builder(); - newProject = builder - .withProjectDir(project.rootDir) - .withGradleUserHomeDir(project.gradle.gradleUserHomeDir) - .withName(project.name) - .build(); - - // need this for gradle to evaluate the project - // and load the different plugins and extensions - newProject.evaluate(); - } catch (Exception e) { - throw new PluginExecutionException("Could not parse build.gradle " + e.getMessage()); - } - } - createNewInstallFeatureUtil(pluginListedEsas, propertiesList, openLibertyVerion, containerName, additionalJsons) + //if installing userFeature, recompile gradle to find mavenLocal artifacts created by prepareFeature task. + if (project.configurations.featuresBom.dependencies) { + try { + ProjectBuilder builder = ProjectBuilder.builder(); + newProject = builder + .withProjectDir(project.rootDir) + .withGradleUserHomeDir(project.gradle.gradleUserHomeDir) + .withName(project.name) + .build(); + + // need this for gradle to evaluate the project + // and load the different plugins and extensions + newProject.evaluate(); + } catch (Exception e) { + throw new PluginExecutionException("Could not parse build.gradle " + e.getMessage()); + } + } + createNewInstallFeatureUtil(pluginListedEsas, propertiesList, openLibertyVerion, containerName, additionalJsons) return util } + } \ No newline at end of file diff --git a/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractServerTask.groovy b/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractServerTask.groovy index a094e7dfe..49b81c5be 100644 --- a/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractServerTask.groovy +++ b/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractServerTask.groovy @@ -38,7 +38,7 @@ import org.apache.commons.io.FilenameUtils import org.apache.commons.io.filefilter.FileFilterUtils import io.openliberty.tools.ant.ServerTask -import io.openliberty.tools.common.plugins.config.ServerConfigDropinXmlDocument; +import io.openliberty.tools.common.plugins.config.ServerConfigXmlDocument; import java.util.ArrayList import java.util.List @@ -927,7 +927,7 @@ abstract class AbstractServerTask extends AbstractLibertyTask { private void writeConfigDropinsServerVariables(File file, Properties varProps, Properties varProjectProps, boolean isDefaultVar) throws IOException, TransformerException, ParserConfigurationException { - ServerConfigDropinXmlDocument configDocument = ServerConfigDropinXmlDocument.newInstance() + ServerConfigXmlDocument configDocument = ServerConfigXmlDocument.newInstance() configDocument.createComment(HEADER) diff --git a/src/main/groovy/io/openliberty/tools/gradle/tasks/DevTask.groovy b/src/main/groovy/io/openliberty/tools/gradle/tasks/DevTask.groovy index a5e80c2d0..58f01dd83 100644 --- a/src/main/groovy/io/openliberty/tools/gradle/tasks/DevTask.groovy +++ b/src/main/groovy/io/openliberty/tools/gradle/tasks/DevTask.groovy @@ -46,7 +46,7 @@ import java.util.concurrent.TimeUnit import java.util.Map.Entry import java.nio.file.Path; -class DevTask extends AbstractServerTask { +class DevTask extends AbstractFeatureTask { private static final String LIBERTY_HOSTNAME = "liberty.hostname"; private static final String LIBERTY_HTTP_PORT = "liberty.http.port"; @@ -80,6 +80,7 @@ class DevTask extends AbstractServerTask { private static final boolean DEFAULT_CONTAINER = false; private static final boolean DEFAULT_SKIP_DEFAULT_PORTS = false; private static final boolean DEFAULT_KEEP_TEMP_DOCKERFILE = false; + private static final boolean DEFAULT_GENERATE_FEATURES = true; protected final String CONTAINER_PROPERTY_ARG = '-P'+CONTAINER_PROPERTY+'=true'; @@ -250,6 +251,16 @@ class DevTask extends AbstractServerTask { this.keepTempDockerfile = keepTempDockerfile; } + @Optional + @Input + Boolean generateFeatures; + + // Need to use a string value to allow someone to specify --generateFeatures=false, if not explicitly set defaults to true + @Option(option = 'generateFeatures', description = 'If true, scan the application binary files to determine which Liberty features should be used. The default value is true.') + void setGenerateFeatures(String generateFeatures) { + this.generateFeatures = Boolean.parseBoolean(generateFeatures); + } + @Optional @Input Boolean clean; @@ -282,17 +293,18 @@ class DevTask extends AbstractServerTask { boolean hotTests, boolean skipTests, String artifactId, int serverStartTimeout, int verifyAppStartTimeout, int appUpdateTimeout, double compileWait, boolean libertyDebug, boolean pollingTest, boolean container, File dockerfile, File dockerBuildContext, - String dockerRunOpts, int dockerBuildTimeout, boolean skipDefaultPorts, boolean keepTempDockerfile, String mavenCacheLocation, String packagingType, File buildFile + String dockerRunOpts, int dockerBuildTimeout, boolean skipDefaultPorts, boolean keepTempDockerfile, + String mavenCacheLocation, String packagingType, File buildFile, boolean generateFeatures ) throws IOException { super(buildDir, serverDirectory, sourceDirectory, testSourceDirectory, configDirectory, projectDirectory, /* multi module project directory */ projectDirectory, resourceDirs, hotTests, skipTests, false /* skipUTs */, false /* skipITs */, artifactId, serverStartTimeout, verifyAppStartTimeout, appUpdateTimeout, ((long) (compileWait * 1000L)), libertyDebug, true /* useBuildRecompile */, true /* gradle */, pollingTest, container, dockerfile, dockerBuildContext, dockerRunOpts, dockerBuildTimeout, skipDefaultPorts, - null /* compileOptions not needed since useBuildRecompile is true */, keepTempDockerfile, mavenCacheLocation, null /* multi module upstream projects */, - false /* recompileDependencies only supported in ci.maven */, packagingType, buildFile, null /* parent build files */, null /* compileArtifactPaths */, null /* testArtifactPaths */, new ArrayList() /* webResources */ + null /* compileOptions not needed since useBuildRecompile is true */, keepTempDockerfile, mavenCacheLocation, null /* multi module upstream projects */, + false /* recompileDependencies only supported in ci.maven */, packagingType, buildFile, null /* parent build files */, generateFeatures, null /* compileArtifactPaths */, null /* testArtifactPaths */, new ArrayList() /* webResources */ ); - ServerFeature servUtil = getServerFeatureUtil(); + ServerFeatureUtil servUtil = getServerFeatureUtil(); this.libertyDirPropertyFiles = AbstractServerTask.getLibertyDirectoryPropertyFiles(installDirectory, userDirectory, serverDirectory); this.existingFeatures = servUtil.getServerFeatures(serverDirectory, libertyDirPropertyFiles); @@ -543,9 +555,9 @@ class DevTask extends AbstractServerTask { } } - if (restartServer) { // - stop Server + // - generate features (if generateFeatures=true) // - create server or runBoostMojo // - install feature // - deploy app @@ -553,6 +565,15 @@ class DevTask extends AbstractServerTask { util.restartServer(); return true; } else if (installFeatures) { + if (generateFeatures) { + // Increment generate features on build dependency change + ProjectConnection gradleConnection = initGradleProjectConnection(); + BuildLauncher gradleBuildLauncher = gradleConnection.newBuild(); + runGradleTask(gradleBuildLauncher, 'compileJava', 'processResources'); // ensure class files exist + Collection javaSourceClassPaths = getJavaSourceClassPaths(); + libertyGenerateFeatures(javaSourceClassPaths, false); + libertyCreate(); // need to run create in order to copy generated config file to target + } libertyInstallFeature(); } @@ -620,7 +641,7 @@ class DevTask extends AbstractServerTask { @Override public void checkConfigFile(File configFile, File serverDir) { - ServerFeature servUtil = getServerFeatureUtil(); + ServerFeatureUtil servUtil = getServerFeatureUtil(); Set features = servUtil.getServerFeatures(serverDir, libertyDirPropertyFiles); if (features == null) { @@ -754,6 +775,29 @@ class DevTask extends AbstractServerTask { } } + @Override + public boolean libertyGenerateFeatures(Collection classes, boolean optimize) { + ProjectConnection gradleConnection = initGradleProjectConnection(); + BuildLauncher gradleBuildLauncher = gradleConnection.newBuild(); + + try { + List options = new ArrayList(); + classes.each { + // generate features for only the classFiles passed (if any) + options.add("--classFile=" + it); + } + options.add("--optimize=" + optimize); + runGenerateFeaturesTask(gradleBuildLauncher, options); + return true; // successfully generated features + } catch (BuildException e) { + // log as error instead of throwing an exception so we do not flood console with stacktrace + logger.error(e.getMessage() + ".\n To disable the automatic generation of features, type 'g' and press Enter."); + return false; + } finally { + gradleConnection.close(); + } + } + @Override public void libertyInstallFeature() { ProjectConnection gradleConnection = initGradleProjectConnection(); @@ -843,6 +887,24 @@ class DevTask extends AbstractServerTask { runGradleTask(gradleBuildLauncher, tasks); } + public void runGenerateFeaturesTask(BuildLauncher gradleBuildLauncher, boolean optimize) throws BuildException { + List options = new ArrayList(); + options.add("--optimize="+optimize); + runGenerateFeaturesTask(gradleBuildLauncher, options); + } + + public void runGenerateFeaturesTask(BuildLauncher gradleBuildLauncher, List options) throws BuildException { + String[] tasks = new String[options != null ? options.size() + 1 : 1]; + tasks[0] = 'generateFeatures'; + if (options != null) { + for(int i = 0; i < options.size(); i++) { + tasks[i+1] = options.get(i); + } + } + + runGradleTask(gradleBuildLauncher, tasks); + } + // If a argument has not been set using CLI arguments set a default value // Using the ServerExtension properties if available, otherwise use hardcoded defaults private void initializeDefaultValues() throws Exception { @@ -890,6 +952,10 @@ class DevTask extends AbstractServerTask { pollingTest = DEFAULT_POLLING_TEST; } + if (generateFeatures == null) { + generateFeatures = DEFAULT_GENERATE_FEATURES; + } + processContainerParams(); } @@ -953,8 +1019,17 @@ class DevTask extends AbstractServerTask { :war :deploy */ + if (generateFeatures) { + // Optimize generate features on startup + runGradleTask(gradleBuildLauncher, 'compileJava', 'processResources'); // ensure class files exist + try { + runGenerateFeaturesTask(gradleBuildLauncher, true); + } catch (BuildException e) { + throw new BuildException(e.getCause().getMessage() + " To disable the automatic generation of features, start dev mode with --generateFeatures=false.", e); + } + } if (!container) { - addLibertyRuntimeProperties(gradleBuildLauncher); + addLibertyRuntimeProperties(gradleBuildLauncher); runGradleTask(gradleBuildLauncher, 'libertyCreate'); // suppress extra install feature warnings (one would have shown up already from the libertyCreate task on the line above) gradleBuildLauncher.addArguments("-D" + DevUtil.SKIP_BETA_INSTALL_WARNING + "=" + Boolean.TRUE.toString()); @@ -979,7 +1054,8 @@ class DevTask extends AbstractServerTask { resourceDirs, hotTests.booleanValue(), skipTests.booleanValue(), artifactId, serverStartTimeout.intValue(), verifyAppStartTimeout.intValue(), verifyAppStartTimeout.intValue(), compileWait.doubleValue(), libertyDebug.booleanValue(), pollingTest.booleanValue(), container.booleanValue(), dockerfile, dockerBuildContext, dockerRunOpts, - dockerBuildTimeout, skipDefaultPorts.booleanValue(), keepTempDockerfile.booleanValue(), localMavenRepoForFeatureUtility, getPackagingType(), buildFile + dockerBuildTimeout, skipDefaultPorts.booleanValue(), keepTempDockerfile.booleanValue(), localMavenRepoForFeatureUtility, + getPackagingType(), buildFile, generateFeatures.booleanValue() ); util.addShutdownHook(executor); @@ -1120,46 +1196,4 @@ class DevTask extends AbstractServerTask { buildLauncher.run(); } - private static ServerFeature serverFeatureUtil; - - private ServerFeature getServerFeatureUtil() { - if (serverFeatureUtil == null) { - serverFeatureUtil = new ServerFeature(); - } - return serverFeatureUtil; - } - - private class ServerFeature extends ServerFeatureUtil { - - @Override - public void debug(String msg) { - logger.debug(msg); - } - - @Override - public void debug(String msg, Throwable e) { - logger.debug(msg, (Throwable) e); - } - - @Override - public void debug(Throwable e) { - logger.debug("Exception received: "+e.getMessage(), (Throwable) e); - } - - @Override - public void warn(String msg) { - logger.warn(msg); - } - - @Override - public void info(String msg) { - logger.info(msg); - } - - @Override - public void error(String msg, Throwable e) { - logger.error(msg, e); - } - - } } diff --git a/src/main/groovy/io/openliberty/tools/gradle/tasks/GenerateFeaturesTask.groovy b/src/main/groovy/io/openliberty/tools/gradle/tasks/GenerateFeaturesTask.groovy new file mode 100644 index 000000000..b85c241dc --- /dev/null +++ b/src/main/groovy/io/openliberty/tools/gradle/tasks/GenerateFeaturesTask.groovy @@ -0,0 +1,348 @@ +/** + * (C) Copyright IBM Corporation 2021. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openliberty.tools.gradle.tasks + + +import io.openliberty.tools.common.plugins.config.ServerConfigXmlDocument +import io.openliberty.tools.common.plugins.config.XmlDocument +import io.openliberty.tools.common.plugins.util.BinaryScannerUtil +import io.openliberty.tools.common.plugins.util.PluginExecutionException +import io.openliberty.tools.common.plugins.util.ServerFeatureUtil +import io.openliberty.tools.gradle.utils.ArtifactDownloadUtil +import org.gradle.api.GradleException +import org.gradle.api.tasks.TaskAction +import org.gradle.api.tasks.options.Option +import org.xml.sax.SAXException +import org.w3c.dom.Element; + +import javax.xml.parsers.ParserConfigurationException +import javax.xml.transform.TransformerException +import java.lang.reflect.InvocationTargetException + +class GenerateFeaturesTask extends AbstractFeatureTask { + + private static final String GENERATED_FEATURES_FILE_NAME = "generated-features.xml"; + protected static final String GENERATED_FEATURES_FILE_PATH = "configDropins/overrides/" + GENERATED_FEATURES_FILE_NAME; + protected static final String FEATURES_FILE_MESSAGE = "The Liberty Gradle Plugin has generated Liberty features necessary for your application in " + GENERATED_FEATURES_FILE_PATH; + protected static final String HEADER = "This file was generated by the Liberty Gradle Plugin and will be overwritten on subsequent runs of the generateFeatures task." + "\n It is recommended that you do not edit this file and that you commit this file to your version control."; + protected static final String GENERATED_FEATURES_COMMENT = "The following features were generated based on API usage detected in your application"; + protected static final String NO_NEW_FEATURES_COMMENT = "No additional features generated"; + + private static final String BINARY_SCANNER_MAVEN_GROUP_ID = "com.ibm.websphere.appmod.tools"; + private static final String BINARY_SCANNER_MAVEN_ARTIFACT_ID = "binary-app-scanner"; + private static final String BINARY_SCANNER_MAVEN_TYPE = "jar"; + private static final String BINARY_SCANNER_MAVEN_VERSION = "[21.0.0.4-SNAPSHOT,)"; + + private static final boolean DEFAULT_OPTIMIZE = true; + + private File binaryScanner; + + GenerateFeaturesTask() { + configure({ + description 'Generate the features used by an application and add to the configuration of a Liberty server' + group 'Liberty' + }) + } + + private List classFiles; + + @Option(option = 'classFile', description = 'If set and optimize is false, will generate features for the list of classes passed.') + void setClassFiles(List classFiles) { + this.classFiles = classFiles; + } + + private Boolean optimize = null; + + // Need to use a string value to allow the ability to specify a value for the parameter (ie. --optimize=false) + @Option(option = 'optimize', description = 'Optimize generating features by passing in all classes and only user specified features.') + void setOptimize(String optimize) { + this.optimize = Boolean.parseBoolean(optimize); + } + + @TaskAction + void generateFeatures() { + binaryScanner = getBinaryScannerJarFromRepository(); + BinaryScannerHandler binaryScannerHandler = new BinaryScannerHandler(binaryScanner); + + if (optimize == null) { + optimize = DEFAULT_OPTIMIZE; + } + + logger.debug("--- Generate Features values ---"); + logger.debug("optimize generate features: " + optimize); + if (classFiles != null && !classFiles.isEmpty()) { + logger.debug("Generate features for the following class files: " + classFiles); + } + + initializeConfigDirectory(); + + // TODO add support for env variables + // commented out for now as the current logic depends on the server dir existing and specifying features with env variables is an edge case + /* def serverDirectory = getServerDir(project); + def libertyDirPropertyFiles; + try { + libertyDirPropertyFiles = getLibertyDirectoryPropertyFiles(getInstallDir(project), getUserDir(project), serverDirectory); + } catch (IOException x) { + logger.debug("Exception reading the server property files", e); + logger.error("Error attempting to generate server feature list. Ensure your user account has read permission to the property files in the server installation directory."); + return; + } */ + + // get existing server features from source directory + ServerFeatureUtil servUtil = getServerFeatureUtil(); + + Set generatedFiles = new HashSet(); + generatedFiles.add(GENERATED_FEATURES_FILE_NAME); + + servUtil.setLowerCaseFeatures(false); + // if optimizing, ignore generated files when passing in existing features to binary scanner + Set existingFeatures = servUtil.getServerFeatures(server.configDirectory, server.serverXmlFile, new HashMap(), optimize ? generatedFiles : null); + if (existingFeatures == null) { + existingFeatures = new HashSet(); + } + logger.debug("Existing features:" + existingFeatures); + servUtil.setLowerCaseFeatures(true); + + Set scannedFeatureList; + try { + Set directories = getClassesDirectories(); + String eeVersion = getEEVersion(project); + String mpVersion = getMPVersion(project); + scannedFeatureList = binaryScannerHandler.runBinaryScanner(existingFeatures, classFiles, directories, eeVersion, mpVersion, optimize); + } catch (BinaryScannerUtil.NoRecommendationException noRecommendation) { + throw new GradleException(String.format(BinaryScannerUtil.BINARY_SCANNER_CONFLICT_MESSAGE3, noRecommendation.getConflicts())); + } catch (BinaryScannerUtil.RecommendationSetException showRecommendation) { + if (showRecommendation.isExistingFeaturesConflict()) { + throw new GradleException(String.format(BinaryScannerUtil.BINARY_SCANNER_CONFLICT_MESSAGE2, showRecommendation.getConflicts(), showRecommendation.getSuggestions())); + } + throw new GradleException(String.format(BinaryScannerUtil.BINARY_SCANNER_CONFLICT_MESSAGE1, showRecommendation.getConflicts(), showRecommendation.getSuggestions())); + } catch (InvocationTargetException | PluginExecutionException x) { + // throw an error when there is a problem not caught in runBinaryScanner() + Object o = x.getCause(); + if (o != null) { + logger.debug("Caused by exception:" + x.getCause().getClass().getName()); + logger.debug("Caused by exception message:" + x.getCause().getMessage()); + } + throw new GradleException("Failed to generate a working set of features. " + x.getMessage(), x); + } + + def missingLibertyFeatures = new HashSet(); + if (scannedFeatureList != null) { + missingLibertyFeatures.addAll(scannedFeatureList); + + servUtil.setLowerCaseFeatures(false); + // get set of user defined features so they can be omitted from the generated file that will be written + Set userDefinedFeatures = optimize ? existingFeatures : servUtil.getServerFeatures(server.configDirectory, server.serverXmlFile, new HashMap(), generatedFiles); + logger.debug("User defined features:" + userDefinedFeatures); + servUtil.setLowerCaseFeatures(true); + if (userDefinedFeatures != null) { + missingLibertyFeatures.removeAll(userDefinedFeatures); + } + } + logger.debug("Features detected by binary scanner which are not in server.xml : " + missingLibertyFeatures); + + def newServerXmlSrc = new File(server.configDirectory, GENERATED_FEATURES_FILE_PATH); + try { + if (missingLibertyFeatures.size() > 0) { + // Create specialized server.xml + ServerConfigXmlDocument configDocument = ServerConfigXmlDocument.newInstance(); + configDocument.createComment(HEADER); + Element featureManagerElem = configDocument.createFeatureManager(); + configDocument.createComment(featureManagerElem, GENERATED_FEATURES_COMMENT); + for (String missing : missingLibertyFeatures) { + logger.debug(String.format("Adding missing feature %s to %s.", missing, GENERATED_FEATURES_FILE_PATH)); + configDocument.createFeature(missing); + } + configDocument.writeXMLDocument(newServerXmlSrc); + logger.debug("Created file " + newServerXmlSrc); + // Add a reference to this new file in existing server.xml. + def serverXml = findConfigFile("server.xml", server.serverXmlFile); + def doc = getServerXmlDocFromConfig(serverXml); + logger.debug("Xml document we'll try to update after generate features doc=" + doc + " file=" + serverXml); + addGenerationCommentToConfig(doc, serverXml); + + logger.lifecycle("Generated the following features: " + missingLibertyFeatures); + // use logger.lifecycle so that message appears without --info tag on + } else { + logger.lifecycle("No additional features were generated."); + if (newServerXmlSrc.exists()) { + // generated-features.xml exists but no additional features were generated + // create empty features list with comment + ServerConfigXmlDocument configDocument = ServerConfigXmlDocument.newInstance(); + configDocument.createComment(HEADER); + Element featureManagerElem = configDocument.createFeatureManager(); + configDocument.createComment(featureManagerElem, NO_NEW_FEATURES_COMMENT); + configDocument.writeXMLDocument(newServerXmlSrc); + } + } + } catch (ParserConfigurationException | TransformerException | IOException e) { + logger.debug("Exception creating the server features file", e); + throw new GradleException("Automatic generation of features failed. Error attempting to create the " + GENERATED_FEATURES_FILE_NAME + ". Ensure your id has write permission to the server installation directory.", e); + } + } + + /** + * Gets the binary scanner jar file from the local cache. + * Downloads it first from connected repositories such as Maven Central if a newer release is available than the cached version. + * Note: Maven updates artifacts daily by default based on the last updated timestamp. Users should use 'mvn -U' to force updates if needed. + * + * @return The File object of the binary scanner jar in the local cache. + * @throws PluginExecutionException + */ + private File getBinaryScannerJarFromRepository() throws PluginExecutionException { + try { + return ArtifactDownloadUtil.downloadBuildArtifact(project, BINARY_SCANNER_MAVEN_GROUP_ID, BINARY_SCANNER_MAVEN_ARTIFACT_ID, BINARY_SCANNER_MAVEN_TYPE, BINARY_SCANNER_MAVEN_VERSION); + } catch (Exception e) { + throw new PluginExecutionException("Could not retrieve the binary scanner jar. Ensure you have a connection to Maven Central or another repository that contains the jar configured in your build.gradle: " + e.getMessage(), e); + } + } + + /** + * Return specificFile if it exists; otherwise check for a file with the requested name in the + * configDirectory and return it if it exists. Null is returned if a file does not exist in + * either location. + */ + private File findConfigFile(String fileName, File specificFile) { + if (specificFile != null && specificFile.exists()) { + return specificFile; + } + + if (server.configDirectory == null) { + return null; + } + File f = new File(server.configDirectory, fileName); + if (f.exists()) { + return f; + } else { + return null; + } + } + + // Convert a file into a document object + private ServerConfigXmlDocument getServerXmlDocFromConfig(File serverXml) { + if (serverXml == null || !serverXml.exists()) { + return null; + } + try { + return ServerConfigXmlDocument.newInstance(serverXml); + } catch (ParserConfigurationException | SAXException | IOException e) { + logger.debug("Exception creating server.xml object model", e); + } + return null; + } + + /** + * Add a comment to server.xml to warn them we created another file with features in it. + */ + private void addGenerationCommentToConfig(ServerConfigXmlDocument doc, File serverXml) { + if (doc == null) { + return; + } + try { + if (doc.createFMComment(FEATURES_FILE_MESSAGE)) { + doc.writeXMLDocument(serverXml); + XmlDocument.addNewlineBeforeFirstElement(serverXml); + } + } catch (IOException | TransformerException e) { + log.debug("Exception adding comment to server.xml", e); + } + return; + } + + private Set getClassesDirectories() { + Set classesDirectories = new ArrayList(); + project.sourceSets.main.getOutput().getClassesDirs().each { + classesDirectories.add(it.getAbsolutePath()); + } + return classesDirectories; + } + + private getEEVersion(Object project) { + String eeVersion = null + project.configurations.compile.allDependencies.each { + dependency -> + if (dependency.group.equals("io.openliberty.features")) { + if (dependency.name.equals("javaee-7.0")) { + eeVersion = "ee7" + } else if (dependency.name.equals("javaee-8.0")) { + eeVersion = "ee8" + } else if (dependency.name.equals("javaeeClient-7.0")) { + eeVersion = "ee7" + } else if (dependency.name.equals("javaeeClient-8.0")) { + eeVersion = "ee8" + } else if (dependency.name.equals("jakartaee-8.0")) { + eeVersion = "ee8" + } + } else if (dependency.group.equals("jakarta.platform") && + dependency.name.equals("jakarta.jakartaee-api") && + dependency.version.equals("8.0.0")) { + eeVersion = "ee8"; + } + } + return eeVersion; + } + + private getMPVersion(Object project) { + String mpVersion = null + project.configurations.compile.allDependencies.each { + if (it.group.equals("org.eclipse.microprofile") && + it.name.equals("microprofile")) { + if (it.version.startsWith("1")) { + mpVersion = "mp1" + } else if (it.version.startsWith("2")) { + mpVersion = "mp2" + } else if (it.version.startsWith("3")) { + mpVersion = "mp3" + } else if (it.version.startsWith("4")) { + mpVersion = "mp4" + } + } + } + return mpVersion; + } + + // Define the logging functions of the binary scanner handler and make it available in this plugin + private class BinaryScannerHandler extends BinaryScannerUtil { + BinaryScannerHandler(File scannerFile) { + super(scannerFile); + } + + @Override + public void debug(String msg) { + logger.debug(msg); + } + + @Override + public void debug(String msg, Throwable t) { + logger.debug(msg, t); + } + + @Override + public void error(String msg) { + logger.error(msg); + } + + @Override + public void warn(String msg) { + logger.warn(msg); + } + + @Override + public void info(String msg) { + logger.lifecycle(msg); + } + } +} diff --git a/src/main/groovy/io/openliberty/tools/gradle/utils/ArtifactDownloadUtil.groovy b/src/main/groovy/io/openliberty/tools/gradle/utils/ArtifactDownloadUtil.groovy new file mode 100644 index 000000000..bfefc78f6 --- /dev/null +++ b/src/main/groovy/io/openliberty/tools/gradle/utils/ArtifactDownloadUtil.groovy @@ -0,0 +1,57 @@ +/** + * (C) Copyright IBM Corporation 2021. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openliberty.tools.gradle.utils + +import org.gradle.api.Project +import org.gradle.api.artifacts.ResolveException +import io.openliberty.tools.common.plugins.util.PluginExecutionException + +public class ArtifactDownloadUtil { + + public static File downloadArtifact(Project project, String groupId, String artifactId, String type, String version) throws PluginExecutionException { + String coordinates = groupId + ":" + artifactId + ":" + version + "@" + type + def dep = project.dependencies.create(coordinates) + def config = project.configurations.detachedConfiguration(dep) + + return downloadFile(project, config, coordinates) + } + + public static File downloadBuildArtifact(Project project, String groupId, String artifactId, String type, String version) throws PluginExecutionException { + String coordinates = groupId + ":" + artifactId + ":" + version + "@" + type + def dep = project.buildscript.dependencies.create(coordinates) + def config = project.buildscript.configurations.detachedConfiguration(dep) + + return downloadFile(project, config, coordinates) + } + + private static File downloadFile(project, config, coordinates) { + Set files = new HashSet() + try { + config.resolvedConfiguration.resolvedArtifacts.each { artifact -> + File artifactFile = artifact.file + files.add(artifactFile) + project.getLogger().debug(artifactFile.toString()) + } + } catch (ResolveException e) { + throw new PluginExecutionException("Could not find artifact with coordinates " + coordinates, e) + } + + if (!files) { + throw new PluginExecutionException("Could not find artifact with coordinates " + coordinates) + } + return files.iterator().next() + } +} \ No newline at end of file diff --git a/src/test/groovy/io/openliberty/tools/gradle/DevTest.groovy b/src/test/groovy/io/openliberty/tools/gradle/DevTest.groovy index ab53e8efd..d3c86d4af 100644 --- a/src/test/groovy/io/openliberty/tools/gradle/DevTest.groovy +++ b/src/test/groovy/io/openliberty/tools/gradle/DevTest.groovy @@ -117,6 +117,7 @@ class DevTest extends AbstractIntegrationTest { if (params != null) { command.append(" " + params); } + command.append(" " + "--generateFeatures=false"); // TODO add feature generation tests System.out.println("Running command: " + command.toString()); ProcessBuilder builder = buildProcess(command.toString()); @@ -246,7 +247,7 @@ class DevTest extends AbstractIntegrationTest { @Test public void manualTestsInvocationTest() throws Exception { - assertFalse(checkLogMessage(2000, "To run tests on demand, press Enter.")); +// assertFalse(checkLogMessage(2000, "To run tests on demand, press Enter.")); writer.write("\n"); writer.flush(); diff --git a/src/test/resources/dev-test/basic-dev-project/build.gradle b/src/test/resources/dev-test/basic-dev-project/build.gradle index 983dac486..acb53d507 100644 --- a/src/test/resources/dev-test/basic-dev-project/build.gradle +++ b/src/test/resources/dev-test/basic-dev-project/build.gradle @@ -12,6 +12,10 @@ buildscript { repositories { mavenLocal() mavenCentral() + // sonatype repo for getting the latest binary scanner jar snapshot + maven { + url "https://oss.sonatype.org/content/repositories" + } } dependencies { classpath "io.openliberty.tools:liberty-gradle-plugin:$lgpVersion"