From 48a30a793b1033ecda5e233b6d34ea4da29ba959 Mon Sep 17 00:00:00 2001 From: Cheryl King Date: Thu, 1 Jun 2023 14:36:22 -0500 Subject: [PATCH 1/3] Preserve keystore_password when merging env properties --- docs/libertyExtensions.md | 5 + .../gradle/tasks/AbstractServerTask.groovy | 34 ++++++- ...stServerEnvKeystorePasswordPreserve.groovy | 93 +++++++++++++++++++ ...stServerEnvKeystorePasswordPreserve.gradle | 92 ++++++++++++++++++ 4 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 src/test/groovy/io/openliberty/tools/gradle/TestServerEnvKeystorePasswordPreserve.groovy create mode 100644 src/test/resources/sample.servlet/testServerEnvKeystorePasswordPreserve.gradle diff --git a/docs/libertyExtensions.md b/docs/libertyExtensions.md index 12c6a634..8ac900a0 100644 --- a/docs/libertyExtensions.md +++ b/docs/libertyExtensions.md @@ -52,3 +52,8 @@ The following properties are supported for server configuration. | timeout | String | 1.0 | Waiting time before the server starts. The default value is 30 seconds. The unit is seconds. Only used with `libertyStart` and `deploy` tasks. | No | | var | Properties | 3.0 | Inline server variables that are written to the `configDropins/overrides/liberty-plugin-variable-config.xml` file in the server directory. The property name is used for the variable `name`, and the property value is used for the variable `value`.| No| | verifyAppStartTimeout | int | 2.0 | Wait time for checking message logs for start of all applications installed with the `deploy` task. Only used with the `libertyStart` task. Default value is 0 seconds with no verification. | No | + +If Liberty configuration is specified with Gradle properties for the server extension properties of type `Properties`, the above indicated files are created in the target Liberty server. By default there is no merging behavior for the Gradle properties with files located in the `configDirectory` or the specific configuration file parameters such as `bootstrapPropertiesFile`, `jvmOptionsFile` and `serverEnvFile`. However, the `liberty.server.env."var"` Gradle properties can be merged with other configured `server.env` files by setting the `mergeServerEnv` parameter to `true`. + +As a special case when `mergeServerEnv` is `false`, an existing `keystore_password` property in the default generated `server.env` file in the target server will be merged in if there is no `serverEnvFile` configured nor `server.env` file located in the `configDirectory`, and the `keystore_password` env var is not defined as a Gradle property. + diff --git a/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractServerTask.groovy b/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractServerTask.groovy index a67baf7a..a9602a91 100644 --- a/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractServerTask.groovy +++ b/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractServerTask.groovy @@ -886,10 +886,14 @@ abstract class AbstractServerTask extends AbstractLibertyTask { private String setServerEnvHelper(File envFile, String serverEnvPath, Properties configuredProps) { if ((server.env != null && !server.env.isEmpty()) || !envProjectProps.isEmpty()) { - if (serverEnvPath != null) { + Properties envPropsToWrite = configuredProps + if (serverEnvPath == null && server.serverEnvFile == null) { + // Do a special case merge but ONLY if there is no server.env file present in configDirectory or specified with serverEnvFile + envPropsToWrite = mergeSpecialPropsFromInstallServerEnvIfAbsent(envFile, configuredProps) + } else if (serverEnvPath != null) { logger.warn("The " + serverEnvPath + " file is overwritten by inlined configuration.") } - writeServerEnvProperties(envFile, configuredProps) + writeServerEnvProperties(envFile, envPropsToWrite) return "inlined configuration" } else if (server.serverEnvFile != null && server.serverEnvFile.exists()) { if (serverEnvPath != null) { @@ -900,6 +904,32 @@ abstract class AbstractServerTask extends AbstractLibertyTask { } } + /** + * Merges envProps with special properties found in envFile, the install (target) server.env. We return a clone/copy of + * envProps, to which any of a list of special properties found in envFile have been added. We give precedence + * to properties already in envProps. + */ + private Properties mergeSpecialPropsFromInstallServerEnvIfAbsent(File envFile, Properties envProps) throws IOException { + + String[] specialProps = { "keystore_password" } + + // Make a copy to avoid side effects + Properties mergedProps = new Properties() + mergedProps.putAll(envProps) + + // From install (target) dir + Properties serverEnvProps = convertServerEnvToProperties(envFile) + + for (String propertyName : specialProps) { + if (serverEnvProps.containsKey(propertyName)) { + mergedProps.putIfAbsent(propertyName,serverEnvProps.get(propertyName)) + } + } + + return mergedProps + } + + private Properties convertServerEnvToProperties(File serverEnv) { Properties serverEnvProps = new Properties(); diff --git a/src/test/groovy/io/openliberty/tools/gradle/TestServerEnvKeystorePasswordPreserve.groovy b/src/test/groovy/io/openliberty/tools/gradle/TestServerEnvKeystorePasswordPreserve.groovy new file mode 100644 index 00000000..7059015b --- /dev/null +++ b/src/test/groovy/io/openliberty/tools/gradle/TestServerEnvKeystorePasswordPreserve.groovy @@ -0,0 +1,93 @@ +package io.openliberty.tools.gradle + +import org.junit.AfterClass +import org.junit.BeforeClass +import org.junit.Test + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; + +import java.io.BufferedReader; +import java.io.FileReader; + + +import org.junit.BeforeClass +import org.junit.FixMethodOrder +import org.junit.Test +import org.junit.runners.MethodSorters +import org.junit.Assert; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; +import org.w3c.dom.Node; +import org.w3c.dom.Element; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +class TestServerEnvKeystorePasswordPreserve extends AbstractIntegrationTest { + static File resourceDir = new File("build/resources/test/sample.servlet") + static File buildDir = new File(integTestDir, "/test-server-env-keystore-password-preserve") + static String buildFilename = "testServerEnvKeystorePasswordPreserve.gradle" + + @BeforeClass + public static void setup() { + createDir(buildDir) + createTestProject(buildDir, resourceDir, buildFilename) + runTasks(buildDir, 'libertyCreate') + } + + @Test + public void check_for_server_env() { + assert new File('build/testBuilds/test-server-env-keystore-password-preserve/build/wlp/usr/servers/LibertyProjectServer/server.env').exists() : 'server.env not found!' + } + + /* + # envProps in build.gradle + env = ['TEST_PROP_2':'white', 'CONFIG_SERVER_ENV_PROPS':'TEST'] + + # default server.env + keystore_password=sfKRrA1ioLdtIFQC9bEfkua + + # Final server.env + # Generated by liberty-gradle-plugin + keystore_password=sfKRrA1ioLdtIFQC9bEfkua + CONFIG_SERVER_ENV_PROPS=TEST + TEST_PROP_2=white + */ + @Test + public void check_server_env_contents() { + File serverEnv = new File("build/testBuilds/test-server-env-keystore-password-preserve/build/wlp/usr/servers/LibertyProjectServer/server.env") + FileInputStream input = new FileInputStream(serverEnv) + + Map serverEnvContents = new HashMap(); + + BufferedReader bf = new BufferedReader(new FileReader(serverEnv)) + String line = bf.readLine(); + boolean commentFound = false + while(line != null) { + //ignore comment lines + if(!line.startsWith("#")) { + String[] keyValuePair = line.split("="); + String key = keyValuePair[0]; + String value = keyValuePair[1]; + + serverEnvContents.put(key,value); + } else { + commentFound = true + Assert.assertTrue("File should have been generated by liberty-gradle-plugin", line.contains("liberty-gradle-plugin")); + } + line = bf.readLine(); + } + Assert.assertTrue("Expected generated by liberty-gradle-plugin comment not found", commentFound); + + // The contents of the default server.env can change over time. + // Only the keystore_password should be preserved at this time. Ensure that no additional lines are in the "merged" server.env file. + Assert.assertEquals("Number of env properties should be 3, but is "+serverEnvContents.size(), serverEnvContents.size(), 3) + Assert.assertTrue("keystore_password mapping found", serverEnvContents.containsKey("keystore_password")) + Assert.assertTrue("CONFIG_SERVER_ENV_PROPS=TEST", serverEnvContents.get("CONFIG_SERVER_ENV_PROPS").equals("TEST")) + Assert.assertTrue("TEST_PROP_2=white", serverEnvContents.get("TEST_PROP_2").equals("white")) + + } + +} \ No newline at end of file diff --git a/src/test/resources/sample.servlet/testServerEnvKeystorePasswordPreserve.gradle b/src/test/resources/sample.servlet/testServerEnvKeystorePasswordPreserve.gradle new file mode 100644 index 00000000..5987ae0a --- /dev/null +++ b/src/test/resources/sample.servlet/testServerEnvKeystorePasswordPreserve.gradle @@ -0,0 +1,92 @@ +/* + This test checks whether the application was successfully installed without the version number in the package + when deploy is called and stripVersion is set to true. +*/ +group = 'liberty.gradle' +version = '1' + +buildscript { + repositories { + mavenLocal() + mavenCentral() + maven { + name = 'Sonatype Nexus Snapshots' + url = 'https://oss.sonatype.org/content/repositories/snapshots/' + } + } + dependencies { + classpath "io.openliberty.tools:liberty-gradle-plugin:$lgpVersion" + } +} + +apply plugin: 'war' +apply plugin: 'liberty' + +sourceCompatibility = 1.7 +targetCompatibility = 1.7 + +compileJava.options.encoding = 'UTF-8' + +ext { + // Liberty server properties + wlpServerName = 'LibertyProjectServer' + serverDirectory = "${project.buildDir}/wlp/usr/servers/${wlpServerName}" + testServerHttpPort = 9080 + testServerHttpsPort = 9443 + + // This is set in the ibm-web-ext.xml file + warContext = 'myLibertyApp' + +} + +liberty { + server{ + serverXmlFile = file("src/main/liberty/config/server-apps-test.xml") + name = wlpServerName + deploy { + apps = [war] + copyLibsDirectory = file("${project.buildDir}/libs") + } + mergeServerEnv = true + env = ['TEST_PROP_2':'white', 'CONFIG_SERVER_ENV_PROPS':'TEST'] + } +} + +repositories { + mavenCentral() +} + +dependencies { + testImplementation 'junit:junit:4.13.1' + providedCompile group: 'javax.servlet', name: 'javax.servlet-api', version:'3.1.0' + implementation 'org.apache.commons:commons-text:1.1' + libertyRuntime group: runtimeGroup, name: kernelArtifactId, version: runtimeVersion +} + +test { + println 'inside the test block' + reports.html.outputLocation = file("$buildDir/reports/unit") + reports.junitXml.outputLocation = file("$buildDir/test-results/unit") + exclude '**/it/**' +} + +task integrationTest(type: Test) { + group 'Verification' + description 'Runs the integration tests.' + reports.html.outputLocation = file("$buildDir/reports/it") + reports.junitXml.outputLocation = file("$buildDir/test-results/it") + include '**/it/**' + exclude '**/unit/**' + + systemProperties = ['liberty.test.port': testServerHttpPort, 'war.name': warContext] +} + +task printMessageAboutRunningServer { + doLast { + println "The server is now running at http://localhost:${testServerHttpPort}/${warContext}" + println "To stop the server run 'gradle libertyStop'" + } +} + +deploy.dependsOn 'war' +libertyStart.finalizedBy 'printMessageAboutRunningServer' From ae6b4442ae6f8dce203abcb707bc1ba28417bbee Mon Sep 17 00:00:00 2001 From: Cheryl King Date: Thu, 1 Jun 2023 17:28:16 -0500 Subject: [PATCH 2/3] Fix previous changes for preserving keystore_password --- .../tools/gradle/tasks/AbstractServerTask.groovy | 10 ++++------ .../testServerEnvKeystorePasswordPreserve.gradle | 1 - 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractServerTask.groovy b/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractServerTask.groovy index a9602a91..5fbedf11 100644 --- a/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractServerTask.groovy +++ b/src/main/groovy/io/openliberty/tools/gradle/tasks/AbstractServerTask.groovy @@ -890,6 +890,7 @@ abstract class AbstractServerTask extends AbstractLibertyTask { if (serverEnvPath == null && server.serverEnvFile == null) { // Do a special case merge but ONLY if there is no server.env file present in configDirectory or specified with serverEnvFile envPropsToWrite = mergeSpecialPropsFromInstallServerEnvIfAbsent(envFile, configuredProps) + logger.warn("The default " + envFile.getCanonicalPath() + " file is overwritten by inlined configuration.") } else if (serverEnvPath != null) { logger.warn("The " + serverEnvPath + " file is overwritten by inlined configuration.") } @@ -911,8 +912,6 @@ abstract class AbstractServerTask extends AbstractLibertyTask { */ private Properties mergeSpecialPropsFromInstallServerEnvIfAbsent(File envFile, Properties envProps) throws IOException { - String[] specialProps = { "keystore_password" } - // Make a copy to avoid side effects Properties mergedProps = new Properties() mergedProps.putAll(envProps) @@ -920,10 +919,9 @@ abstract class AbstractServerTask extends AbstractLibertyTask { // From install (target) dir Properties serverEnvProps = convertServerEnvToProperties(envFile) - for (String propertyName : specialProps) { - if (serverEnvProps.containsKey(propertyName)) { - mergedProps.putIfAbsent(propertyName,serverEnvProps.get(propertyName)) - } + String propertyName = "keystore_password" + if (serverEnvProps.containsKey(propertyName)) { + mergedProps.putIfAbsent(propertyName,serverEnvProps.get(propertyName)) } return mergedProps diff --git a/src/test/resources/sample.servlet/testServerEnvKeystorePasswordPreserve.gradle b/src/test/resources/sample.servlet/testServerEnvKeystorePasswordPreserve.gradle index 5987ae0a..61433492 100644 --- a/src/test/resources/sample.servlet/testServerEnvKeystorePasswordPreserve.gradle +++ b/src/test/resources/sample.servlet/testServerEnvKeystorePasswordPreserve.gradle @@ -47,7 +47,6 @@ liberty { apps = [war] copyLibsDirectory = file("${project.buildDir}/libs") } - mergeServerEnv = true env = ['TEST_PROP_2':'white', 'CONFIG_SERVER_ENV_PROPS':'TEST'] } } From 9123632e6858e799d2103cacb53d4fc16c7382d9 Mon Sep 17 00:00:00 2001 From: Cheryl King Date: Thu, 1 Jun 2023 18:39:09 -0500 Subject: [PATCH 3/3] Fix existing test case and remove unnecessary new test case --- .../TestCreateWithInlineProperties.groovy | 10 +- ...stServerEnvKeystorePasswordPreserve.groovy | 93 ------------------- ...stServerEnvKeystorePasswordPreserve.gradle | 91 ------------------ 3 files changed, 7 insertions(+), 187 deletions(-) delete mode 100644 src/test/groovy/io/openliberty/tools/gradle/TestServerEnvKeystorePasswordPreserve.groovy delete mode 100644 src/test/resources/sample.servlet/testServerEnvKeystorePasswordPreserve.gradle diff --git a/src/test/groovy/io/openliberty/tools/gradle/TestCreateWithInlineProperties.groovy b/src/test/groovy/io/openliberty/tools/gradle/TestCreateWithInlineProperties.groovy index 883ea56f..455c7f6f 100644 --- a/src/test/groovy/io/openliberty/tools/gradle/TestCreateWithInlineProperties.groovy +++ b/src/test/groovy/io/openliberty/tools/gradle/TestCreateWithInlineProperties.groovy @@ -72,11 +72,15 @@ public class TestCreateWithInlineProperties extends AbstractIntegrationTest{ prop = new Properties(); prop.load( input2 ); - assert prop.size() == 2 : "expected 2 properties in server.env file but found "+prop.size() + assert prop.size() == 3 : "expected 3 properties in server.env file but found "+prop.size() String value2 = prop.getProperty("some.env.var"); - assert value2 != null && value2.equals("someValue") : "property not found in server.env file" + assert value2 != null && value2.equals("someValue") : "some.env.var property not found in server.env file" value2 = prop.getProperty("another.env.var"); - assert value2 != null && value2.equals("anotherValue") : "property not found in server.env file" + assert value2 != null && value2.equals("anotherValue") : "another.env.var property not found in server.env file" + // if no server.env file is specified in configuration and mergeServerEnv is not set to true, + // the keystore_password is still preserved when present in the default server.env file and not specified in a property + value2 = prop.getProperty("keystore_password"); + assert value2 != null : "keystore_password property not found in server.env file" assert libertyConfigVarOverridesFile.text.contains("name=\"someVar\"") : "configDropins/overrides/liberty-plugin-variable-config.xml does not contain expected variable: "+libertyConfigVarOverridesFile.text assert libertyConfigVarOverridesFile.text.contains("value=\"someValue\"") : "configDropins/overrides/liberty-plugin-variable-config.xml does not contain expected variable: "+libertyConfigVarOverridesFile.text diff --git a/src/test/groovy/io/openliberty/tools/gradle/TestServerEnvKeystorePasswordPreserve.groovy b/src/test/groovy/io/openliberty/tools/gradle/TestServerEnvKeystorePasswordPreserve.groovy deleted file mode 100644 index 7059015b..00000000 --- a/src/test/groovy/io/openliberty/tools/gradle/TestServerEnvKeystorePasswordPreserve.groovy +++ /dev/null @@ -1,93 +0,0 @@ -package io.openliberty.tools.gradle - -import org.junit.AfterClass -import org.junit.BeforeClass -import org.junit.Test - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathFactory; - -import java.io.BufferedReader; -import java.io.FileReader; - - -import org.junit.BeforeClass -import org.junit.FixMethodOrder -import org.junit.Test -import org.junit.runners.MethodSorters -import org.junit.Assert; -import org.w3c.dom.Document; -import org.w3c.dom.NodeList; -import org.w3c.dom.Node; -import org.w3c.dom.Element; - -@FixMethodOrder(MethodSorters.NAME_ASCENDING) -class TestServerEnvKeystorePasswordPreserve extends AbstractIntegrationTest { - static File resourceDir = new File("build/resources/test/sample.servlet") - static File buildDir = new File(integTestDir, "/test-server-env-keystore-password-preserve") - static String buildFilename = "testServerEnvKeystorePasswordPreserve.gradle" - - @BeforeClass - public static void setup() { - createDir(buildDir) - createTestProject(buildDir, resourceDir, buildFilename) - runTasks(buildDir, 'libertyCreate') - } - - @Test - public void check_for_server_env() { - assert new File('build/testBuilds/test-server-env-keystore-password-preserve/build/wlp/usr/servers/LibertyProjectServer/server.env').exists() : 'server.env not found!' - } - - /* - # envProps in build.gradle - env = ['TEST_PROP_2':'white', 'CONFIG_SERVER_ENV_PROPS':'TEST'] - - # default server.env - keystore_password=sfKRrA1ioLdtIFQC9bEfkua - - # Final server.env - # Generated by liberty-gradle-plugin - keystore_password=sfKRrA1ioLdtIFQC9bEfkua - CONFIG_SERVER_ENV_PROPS=TEST - TEST_PROP_2=white - */ - @Test - public void check_server_env_contents() { - File serverEnv = new File("build/testBuilds/test-server-env-keystore-password-preserve/build/wlp/usr/servers/LibertyProjectServer/server.env") - FileInputStream input = new FileInputStream(serverEnv) - - Map serverEnvContents = new HashMap(); - - BufferedReader bf = new BufferedReader(new FileReader(serverEnv)) - String line = bf.readLine(); - boolean commentFound = false - while(line != null) { - //ignore comment lines - if(!line.startsWith("#")) { - String[] keyValuePair = line.split("="); - String key = keyValuePair[0]; - String value = keyValuePair[1]; - - serverEnvContents.put(key,value); - } else { - commentFound = true - Assert.assertTrue("File should have been generated by liberty-gradle-plugin", line.contains("liberty-gradle-plugin")); - } - line = bf.readLine(); - } - Assert.assertTrue("Expected generated by liberty-gradle-plugin comment not found", commentFound); - - // The contents of the default server.env can change over time. - // Only the keystore_password should be preserved at this time. Ensure that no additional lines are in the "merged" server.env file. - Assert.assertEquals("Number of env properties should be 3, but is "+serverEnvContents.size(), serverEnvContents.size(), 3) - Assert.assertTrue("keystore_password mapping found", serverEnvContents.containsKey("keystore_password")) - Assert.assertTrue("CONFIG_SERVER_ENV_PROPS=TEST", serverEnvContents.get("CONFIG_SERVER_ENV_PROPS").equals("TEST")) - Assert.assertTrue("TEST_PROP_2=white", serverEnvContents.get("TEST_PROP_2").equals("white")) - - } - -} \ No newline at end of file diff --git a/src/test/resources/sample.servlet/testServerEnvKeystorePasswordPreserve.gradle b/src/test/resources/sample.servlet/testServerEnvKeystorePasswordPreserve.gradle deleted file mode 100644 index 61433492..00000000 --- a/src/test/resources/sample.servlet/testServerEnvKeystorePasswordPreserve.gradle +++ /dev/null @@ -1,91 +0,0 @@ -/* - This test checks whether the application was successfully installed without the version number in the package - when deploy is called and stripVersion is set to true. -*/ -group = 'liberty.gradle' -version = '1' - -buildscript { - repositories { - mavenLocal() - mavenCentral() - maven { - name = 'Sonatype Nexus Snapshots' - url = 'https://oss.sonatype.org/content/repositories/snapshots/' - } - } - dependencies { - classpath "io.openliberty.tools:liberty-gradle-plugin:$lgpVersion" - } -} - -apply plugin: 'war' -apply plugin: 'liberty' - -sourceCompatibility = 1.7 -targetCompatibility = 1.7 - -compileJava.options.encoding = 'UTF-8' - -ext { - // Liberty server properties - wlpServerName = 'LibertyProjectServer' - serverDirectory = "${project.buildDir}/wlp/usr/servers/${wlpServerName}" - testServerHttpPort = 9080 - testServerHttpsPort = 9443 - - // This is set in the ibm-web-ext.xml file - warContext = 'myLibertyApp' - -} - -liberty { - server{ - serverXmlFile = file("src/main/liberty/config/server-apps-test.xml") - name = wlpServerName - deploy { - apps = [war] - copyLibsDirectory = file("${project.buildDir}/libs") - } - env = ['TEST_PROP_2':'white', 'CONFIG_SERVER_ENV_PROPS':'TEST'] - } -} - -repositories { - mavenCentral() -} - -dependencies { - testImplementation 'junit:junit:4.13.1' - providedCompile group: 'javax.servlet', name: 'javax.servlet-api', version:'3.1.0' - implementation 'org.apache.commons:commons-text:1.1' - libertyRuntime group: runtimeGroup, name: kernelArtifactId, version: runtimeVersion -} - -test { - println 'inside the test block' - reports.html.outputLocation = file("$buildDir/reports/unit") - reports.junitXml.outputLocation = file("$buildDir/test-results/unit") - exclude '**/it/**' -} - -task integrationTest(type: Test) { - group 'Verification' - description 'Runs the integration tests.' - reports.html.outputLocation = file("$buildDir/reports/it") - reports.junitXml.outputLocation = file("$buildDir/test-results/it") - include '**/it/**' - exclude '**/unit/**' - - systemProperties = ['liberty.test.port': testServerHttpPort, 'war.name': warContext] -} - -task printMessageAboutRunningServer { - doLast { - println "The server is now running at http://localhost:${testServerHttpPort}/${warContext}" - println "To stop the server run 'gradle libertyStop'" - } -} - -deploy.dependsOn 'war' -libertyStart.finalizedBy 'printMessageAboutRunningServer'