diff --git a/src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java b/src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java index 1a1a2e4d..d7eee339 100644 --- a/src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java +++ b/src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java @@ -22,6 +22,9 @@ import java.io.InputStream; import java.net.URL; import java.net.URLConnection; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -48,6 +51,7 @@ import org.xml.sax.SAXException; import io.openliberty.tools.common.CommonLoggerI; +import io.openliberty.tools.common.plugins.util.ServerFeatureUtil; import io.openliberty.tools.common.plugins.util.VariableUtility; // Moved from ci.maven/liberty-maven-plugin/src/main/java/net/wasdev/wlp/maven/plugins/ServerConfigDocument.java @@ -116,19 +120,85 @@ public File getServerXML() { return serverXMLFile; } + /** + * Deprecated. Migrate to the simpler constructor. + * @param log + * @param serverXML + * @param configDir + * @param bootstrapFile + * @param bootstrapProp + * @param serverEnvFile + * @param giveConfigDirPrecedence + * @param libertyDirPropertyFiles - Contains a property to file mapping of directory locations + */ public ServerConfigDocument(CommonLoggerI log, File serverXML, File configDir, File bootstrapFile, Map bootstrapProp, File serverEnvFile, boolean giveConfigDirPrecedence, Map libertyDirPropertyFiles) { - initializeAppsLocation(log, serverXML, configDir, bootstrapFile, bootstrapProp, serverEnvFile, giveConfigDirPrecedence, libertyDirPropertyFiles); + this.log = log; + serverXMLFile = serverXML; + configDirectory = configDir; + if (libertyDirPropertyFiles != null) { + libertyDirectoryPropertyToFile = new HashMap(libertyDirPropertyFiles); + if (libertyDirPropertyFiles.containsKey(ServerFeatureUtil.SERVER_CONFIG_DIR)) { + configDirectory = libertyDirPropertyFiles.get(ServerFeatureUtil.SERVER_CONFIG_DIR); + } + } else { + log.warn("The properties for directories are null and could lead to application locations not being resolved correctly."); + libertyDirectoryPropertyToFile = new HashMap(); + } + locations = new HashSet(); + names = new HashSet(); + namelessLocations = new HashSet(); + locationsAndNames = new HashMap(); + props = new Properties(); + defaultProps = new Properties(); + + initializeAppsLocation(); + } + + /** + * Adapt when ready. Expects the libertyDirPropertyFiles to be populated + * @param log + * @param libertyDirPropertyFiles + */ + public ServerConfigDocument(CommonLoggerI log, Map libertyDirPropertyFiles) { + this.log = log; + if (libertyDirPropertyFiles != null) { + libertyDirectoryPropertyToFile = new HashMap(libertyDirPropertyFiles); + configDirectory = libertyDirectoryPropertyToFile.get(ServerFeatureUtil.SERVER_CONFIG_DIR); + serverXMLFile = getFileFromConfigDirectory("server.xml"); + } else { + log.warn("The properties for directories are null and could lead to application locations not being resolved correctly."); + libertyDirectoryPropertyToFile = new HashMap(); + } + locations = new HashSet(); + names = new HashSet(); + namelessLocations = new HashSet(); + locationsAndNames = new HashMap(); + props = new Properties(); + defaultProps = new Properties(); + + initializeAppsLocation(); } // LCLS constructor + // TODO: populate libertyDirectoryPropertyToFile with workspace information public ServerConfigDocument(CommonLoggerI log) { + this(log, null); + } + + // test constructor that takes in initial properties to be called modularly + public ServerConfigDocument(CommonLoggerI log, Map libertyDirPropertyFiles, Properties initProperties) { this.log = log; + libertyDirectoryPropertyToFile = new HashMap(libertyDirPropertyFiles); + configDirectory = libertyDirectoryPropertyToFile.get(ServerFeatureUtil.SERVER_CONFIG_DIR); + serverXMLFile = getFileFromConfigDirectory("server.xml"); + locations = new HashSet(); + names = new HashSet(); + namelessLocations = new HashSet(); + locationsAndNames = new HashMap(); props = new Properties(); + if (initProperties != null) props.putAll(initProperties); defaultProps = new Properties(); - - // TODO: populate with workspace information - libertyDirectoryPropertyToFile = new HashMap(); } private DocumentBuilder getDocumentBuilder() { @@ -151,80 +221,58 @@ private DocumentBuilder getDocumentBuilder() { return docBuilder; } - private void initializeAppsLocation(CommonLoggerI log, File serverXML, File configDir, File bootstrapFile, - Map bootstrapProp, File serverEnvFile, boolean giveConfigDirPrecedence, Map libertyDirPropertyFiles) { + /** + // Server variable precedence in ascending order if defined in multiple locations. + // 1. variable default values in the server.xml file + // 2. environment variables + // server.env + // a. ${wlp.install.dir}/etc/ + // b. ${wlp.user.dir}/shared/ + // c. ${server.config.dir}/ + // jvm.options + // a. ${wlp.user.dir}/shared/jvm.options + // b. ${server.config.dir}/configDropins/defaults/ + // c. ${server.config.dir}/ + // d. ${server.config.dir}/configDropins/overrides/ + // 3. bootstrap.properties + // a. additional references by bootstrap.include + // 4. Java system properties + // 5. Variables loaded from files in the ${server.config.dir}/variables directory or + // other directories as specified by the VARIABLE_SOURCE_DIRS environment variable + // 6. variable values declared in the server.xml file + // a. ${server.config.dir}/configDropins/defaults/ + // b. ${server.config.dir}/server.xml + // c. ${server.config.dir}/configDropins/overrides/ + // 7. variables declared on the command line + */ + public void initializeAppsLocation() { try { - this.log = log; - serverXMLFile = serverXML; - configDirectory = configDir; - if (libertyDirPropertyFiles != null) { - libertyDirectoryPropertyToFile = new HashMap(libertyDirPropertyFiles); - } else { - log.warn("The properties for directories are null and could lead to application locations not being resolved correctly."); - libertyDirectoryPropertyToFile = new HashMap(); - } - - locations = new HashSet(); - names = new HashSet(); - namelessLocations = new HashSet(); - locationsAndNames = new HashMap(); - props = new Properties(); - defaultProps = new Properties(); - + // 1. Need to parse variables in the server.xml for default values before trying to + // find the include files in case one of the variables is used in the location. Document doc = parseDocument(serverXMLFile); - - // Server variable precedence in ascending order if defined in - // multiple locations. - // - // 1. defaultValue from variables defined in server.xml or defined in files - // e.g. - // 2. variables from 'server.env' - // 3. variables from 'bootstrap.properties' - // 4. variables defined in files - // 5. variables from configDropins/defaults/ - // 6. variables defined in server.xml - // e.g. - // 7. variables from configDropins/overrides/ - - // 1. Need to parse variables in the server.xml for default values before trying to find the include files in case one of the variables is used - // in the location. parseVariablesForDefaultValues(doc); // 2. get variables from server.env - File cfgFile = findConfigFile("server.env", serverEnvFile, giveConfigDirPrecedence); + processServerEnv(); - if (cfgFile != null) { - parseProperties(new FileInputStream(cfgFile)); - } + // 3. get variables from jvm.options. Incomplete uncommon usecase. Uncomment when ready. + // processJvmOptions(); // 3. get variables from bootstrap.properties - File cfgDirFile = getFileFromConfigDirectory("bootstrap.properties"); - - if (giveConfigDirPrecedence && cfgDirFile != null) { - parseProperties(new FileInputStream(cfgDirFile)); - } else if (bootstrapProp != null && !bootstrapProp.isEmpty()) { - for (Map.Entry entry : bootstrapProp.entrySet()) { - if (entry.getValue() != null) { - props.setProperty(entry.getKey(),entry.getValue()); - } - } - } else if (bootstrapFile != null && bootstrapFile.exists()) { - parseProperties(new FileInputStream(bootstrapFile)); - } else if (cfgDirFile != null) { - parseProperties(new FileInputStream(cfgDirFile)); - } + processBootstrapProperties(); - // 4. parse variables from include files (both default and non-default values - which we store separately) - parseIncludeVariables(doc); + // 4. Java system properties + processSystemProperties(); - // 5. variables from configDropins/defaults/ - parseConfigDropinsDirVariables("defaults"); + // 5. Variables loaded from 'variables' directory + processVariablesDirectory(); - // 6. variables defined in server.xml - non-default values - parseVariablesForValues(doc); + // 6. variable values declared in server.xml(s) + processServerXml(doc); - // 7. variables from configDropins/overrides/ - parseConfigDropinsDirVariables("overrides"); + // 7. variables declared on the command line + // Maven: https://github.com/OpenLiberty/ci.maven/blob/main/docs/common-server-parameters.md#setting-liberty-configuration-with-maven-project-properties + // Gradle: https://github.com/dshimo/ci.gradle/blob/main/docs/libertyExtensions.md parseApplication(doc, XPATH_SERVER_APPLICATION); parseApplication(doc, XPATH_SERVER_WEB_APPLICATION); @@ -232,12 +280,173 @@ private void initializeAppsLocation(CommonLoggerI log, File serverXML, File conf parseNames(doc, "/server/application | /server/webApplication | /server/enterpriseApplication"); parseInclude(doc); parseConfigDropinsDir(); - } catch (Exception e) { e.printStackTrace(); } } + /** + * server.env file read order + * 1. {wlp.install.dir}/etc/ + * 2. {wlp.user.dir}/shared/ + * 3. {server.config.dir}/ + * @param serverEnvFile + * @throws Exception + * @throws FileNotFoundException + */ + public void processServerEnv() throws Exception, FileNotFoundException { + final String serverEnvString = "server.env"; + parsePropertiesFromFile(new File(libertyDirectoryPropertyToFile.get(ServerFeatureUtil.WLP_INSTALL_DIR), + "etc" + File.separator + serverEnvString)); + parsePropertiesFromFile(new File(libertyDirectoryPropertyToFile.get(ServerFeatureUtil.WLP_USER_DIR), + "shared" + File.separator + serverEnvString)); + parsePropertiesFromFile(getFileFromConfigDirectory(serverEnvString)); + } + + /** + * Likely not needed to be processed by the LMP/LGP tools. These properties benefit the JVM + * System properties would need to process out -D. jvm.options do not support variable substitution + * 1. ${wlp.user.dir}/shared/jvm.options + * 2. ${server.config.dir}/configDropins/defaults/ + * 3. ${server.config.dir}/ + * 4. ${server.config.dir}/configDropins/overrides/ + * @throws FileNotFoundException + * @throws Exception + */ + public void processJvmOptions() throws FileNotFoundException, Exception { + final String jvmOptionsString = "jvm.options"; + parsePropertiesFromFile(new File(libertyDirectoryPropertyToFile.get(ServerFeatureUtil.WLP_USER_DIR), + "shared" + File.separator + jvmOptionsString)); + parsePropertiesFromFile(getFileFromConfigDirectory("configDropins/default/" + jvmOptionsString)); + parsePropertiesFromFile(getFileFromConfigDirectory(jvmOptionsString)); + parsePropertiesFromFile(getFileFromConfigDirectory("configDropins/overrides/" + jvmOptionsString)); + } + + /** + * Process bootstrap.properties and boostrap.include + * @param bootstrapProp - Populated in Maven/Gradle + * @param bootstrapFile - Optional specific file which will take precedence over config dir file + * @throws Exception + * @throws FileNotFoundException + */ + public void processBootstrapProperties() throws Exception, FileNotFoundException { + File bootstrapFile = getFileFromConfigDirectory("bootstrap.properties"); + if (bootstrapFile == null) { + return; + } + + parsePropertiesFromFile(bootstrapFile); + if (props.containsKey("bootstrap.include")) { + Set visited = new HashSet(); + visited.add(bootstrapFile.getAbsolutePath()); + processBootstrapInclude(visited); + } + } + + /** + * Recursive processing for a series of bootstrap.include that terminates upon revisit + * @param bootstrapIncludeLocation + * @param processedBootstrapIncludes + * @throws Exception + * @throws FileNotFoundException + */ + private void processBootstrapInclude(Set processedBootstrapIncludes) throws FileNotFoundException, Exception { + String bootstrapIncludeLocationString = props.getProperty("bootstrap.include"); + Path bootstrapIncludePath = Paths.get(bootstrapIncludeLocationString); + File bootstrapIncludeFile = bootstrapIncludePath.isAbsolute() ? + new File(bootstrapIncludePath.toString()) : new File(configDirectory, bootstrapIncludePath.toString()); + + if (processedBootstrapIncludes.contains(bootstrapIncludeFile.getAbsolutePath())) { + return; + } + + if (bootstrapIncludeFile.exists()) { + parsePropertiesFromFile(bootstrapIncludeFile); + processedBootstrapIncludes.add(bootstrapIncludeFile.getAbsolutePath()); + processBootstrapInclude(processedBootstrapIncludes); + } + } + + private void processSystemProperties() { + props.putAll(System.getProperties()); + } + + /** + * By default, ${server.config.directory}/variables is processed. + * If VARIABLE_SOURCE_DIRS is defined, those directories are processed instead. + * A list of directories are delimited by ';' on Windows, and ':' on Unix + * @throws Exception + * @throws FileNotFoundException + */ + public void processVariablesDirectory() throws FileNotFoundException, Exception { + final String variableDirectoryProperty = "VARIABLE_SOURCE_DIRS"; + + ArrayList toProcess = new ArrayList(); + if (!props.containsKey(variableDirectoryProperty)) { + toProcess.add(getFileFromConfigDirectory("variables")); + } else { + String delimiter = (File.separator.equals("/")) ? ":" : ";"; // OS heuristic + String[] directories = props.get(variableDirectoryProperty).toString().split(delimiter); + for (String directory : directories) { + Path directoryPath = Paths.get(directory); + File directoryFile = directoryPath.toFile(); + if (directoryFile.exists()) { + toProcess.add(directoryFile); + } + } + } + + for (File directory : toProcess) { + if (directory == null || !directory.isDirectory()) { + continue; + } + processVariablesDirectory(directory, ""); + } + } + + /** + * The file name defines the variable name and its contents define the value. + * If a directory is nested within a directory, it is recurisvely processed. + * A nested file will have its parent dir prepended for the property name e.g. {parent directory}/{file name} + * If the file name ends with *.properties, then it's processed as a properties file. + * @param directory - The directory being processed + * @param propertyPrefix - Tracks the nested directories to prepend + * @throws FileNotFoundException + * @throws Exception + */ + private void processVariablesDirectory(File directory, String propertyPrefix) + throws FileNotFoundException, Exception { + for (File child : directory.listFiles()) { + if (child.isDirectory()) { + processVariablesDirectory(child, child.getName() + File.separator); + continue; + } + + if (child.getName().endsWith(".properties")) { + parsePropertiesFromFile(child); + continue; + } + + String propertyName = propertyPrefix + child.getName(); + String propertyValue = new String(Files.readAllBytes(child.toPath())); + props.setProperty(propertyName, propertyValue); + } + } + + /** + * + * @param doc + * @throws XPathExpressionException + * @throws IOException + * @throws SAXException + */ + public void processServerXml(Document doc) throws XPathExpressionException, IOException, SAXException { + parseIncludeVariables(doc); + parseConfigDropinsDirVariables("defaults"); + parseVariablesForValues(doc); + parseConfigDropinsDirVariables("overrides"); + } + //Checks for application names in the document. Will add locations without names to a Set private void parseNames(Document doc, String expression) throws XPathExpressionException, IOException, SAXException { // parse input document @@ -516,6 +725,13 @@ private Document parseDocument(InputStream in) throws SAXException, IOException } } + public void parsePropertiesFromFile(File propertiesFile) throws Exception, FileNotFoundException { + if (propertiesFile != null && propertiesFile.exists()) { + parseProperties(new FileInputStream(propertiesFile)); + log.debug("Processed properties from file: " + propertiesFile.getAbsolutePath()); + } + } + private void parseProperties(InputStream ins) throws Exception { try { props.load(ins); @@ -539,7 +755,7 @@ private boolean isValidURL(String url) { } - private void parseVariablesForDefaultValues(Document doc) throws XPathExpressionException { + public void parseVariablesForDefaultValues(Document doc) throws XPathExpressionException { parseVariables(doc, true, false, false); } @@ -590,24 +806,26 @@ public void parseIncludeVariables(Document doc) throws XPathExpressionException, NodeList nodeList = (NodeList) XPATH_SERVER_INCLUDE.evaluate(doc, XPathConstants.NODESET); for (int i = 0; i < nodeList.getLength(); i++) { - if (nodeList.item(i) instanceof Element) { - Element child = (Element) nodeList.item(i); - // Need to handle more variable substitution for include location. - String nodeValue = child.getAttribute("location"); - String includeFileName = VariableUtility.resolveVariables(log, nodeValue, null, getProperties(), getDefaultProperties(), getLibertyDirPropertyFiles()); + if (!(nodeList.item(i) instanceof Element)) { + continue; + } - if (includeFileName == null || includeFileName.trim().isEmpty()) { - log.warn("Unable to resolve include file location "+nodeValue+". Skipping the included file during application location processing."); - continue; - } + Element child = (Element) nodeList.item(i); + // Need to handle more variable substitution for include location. + String nodeValue = child.getAttribute("location"); + String includeFileName = VariableUtility.resolveVariables(log, nodeValue, null, getProperties(), getDefaultProperties(), getLibertyDirPropertyFiles()); - ArrayList inclDocs = getIncludeDocs(includeFileName); + if (includeFileName == null || includeFileName.trim().isEmpty()) { + log.warn("Unable to resolve include file location "+nodeValue+". Skipping the included file during application location processing."); + continue; + } - for (Document inclDoc : inclDocs) { - parseVariablesForBothValues(inclDoc); - // handle nested include elements - parseIncludeVariables(inclDoc); - } + ArrayList inclDocs = getIncludeDocs(includeFileName); + + for (Document inclDoc : inclDocs) { + parseVariablesForBothValues(inclDoc); + // handle nested include elements + parseIncludeVariables(inclDoc); } } } @@ -658,42 +876,15 @@ private void parseDropinsFilesVariables(File file) } } - /* - * If giveConfigDirPrecedence is set to true, return the file from the configDirectory if it exists; - * otherwise return specificFile if it exists, or null if not. - * If giveConfigDirPrecedence is set to false, return specificFile if it exists; - * otherwise return the file from the configDirectory if it exists, or null if not. - */ - private File findConfigFile(String fileName, File specificFile, boolean giveConfigDirPrecedence) { - File f = new File(configDirectory, fileName); - - if (giveConfigDirPrecedence) { - if (configDirectory != null && f.exists()) { - return f; - } - if (specificFile != null && specificFile.exists()) { - return specificFile; - } - } else { - if (specificFile != null && specificFile.exists()) { - return specificFile; - } - if (configDirectory != null && f.exists()) { - return f; - } - } - - return null; - } - /* * Get the file from configDrectory if it exists, or null if not */ - private File getFileFromConfigDirectory(String file) { - File f = new File(configDirectory, file); + private File getFileFromConfigDirectory(String filename) { + File f = new File(configDirectory, filename); if (configDirectory != null && f.exists()) { return f; } + log.debug(filename + " was not found in: " + configDirectory.getAbsolutePath()); return null; } } diff --git a/src/main/java/io/openliberty/tools/common/plugins/util/VariableUtility.java b/src/main/java/io/openliberty/tools/common/plugins/util/VariableUtility.java index 46d5471f..bf58fa70 100644 --- a/src/main/java/io/openliberty/tools/common/plugins/util/VariableUtility.java +++ b/src/main/java/io/openliberty/tools/common/plugins/util/VariableUtility.java @@ -50,34 +50,33 @@ public static String resolveVariables(CommonLoggerI log, String nodeValue, Colle // Found recursive reference when resolving variables. Log message and return null. log.debug("Found a recursive variable reference when resolving ${" + varName + "}"); return null; - } else { - variablesToResolve.add(varName); } + variablesToResolve.add(varName); } for (String nextVariable : variablesToResolve) { String value = getPropertyValue(nextVariable, props, defaultProps, libDirPropFiles); - if (value != null && !value.isEmpty()) { - Collection thisVariableChain = new HashSet (); - thisVariableChain.add(nextVariable); - - if (variableChain != null && !variableChain.isEmpty()) { - thisVariableChain.addAll(variableChain); - } - - String resolvedValue = resolveVariables(log, value, thisVariableChain, props, defaultProps, libDirPropFiles); - - if (resolvedValue != null) { - String escapedVariable = Matcher.quoteReplacement(nextVariable); - // For Windows, avoid escaping the backslashes in the resolvedValue by changing to forward slashes - resolvedValue = resolvedValue.replace("\\","/"); - resolved = resolved.replaceAll("\\$\\{" + escapedVariable + "\\}", resolvedValue); - } else { - // Variable value could not be resolved. Log message and return null. - log.debug("Could not resolve the value " + value + " for variable ${" + nextVariable + "}"); - return null; - } + if (value == null || value.isEmpty()) { + // Variable could not be resolved. Log message and return null. + log.debug("Variable " + nextVariable + " cannot be resolved."); + return null; + } + + Collection thisVariableChain = new HashSet (); + thisVariableChain.add(nextVariable); + + if (variableChain != null && !variableChain.isEmpty()) { + thisVariableChain.addAll(variableChain); + } + + String resolvedValue = resolveVariables(log, value, thisVariableChain, props, defaultProps, libDirPropFiles); + + if (resolvedValue != null) { + String escapedVariable = Matcher.quoteReplacement(nextVariable); + // For Windows, avoid escaping the backslashes in the resolvedValue by changing to forward slashes + resolvedValue = resolvedValue.replace("\\","/"); + resolved = resolved.replaceAll("\\$\\{" + escapedVariable + "\\}", resolvedValue); } else { // Variable could not be resolved. Log message and return null. log.debug("Variable " + nextVariable + " cannot be resolved."); @@ -90,35 +89,61 @@ public static String resolveVariables(CommonLoggerI log, String nodeValue, Colle return resolved; } - public static String getPropertyValue(String propertyName, Properties props, Properties defaultProps, Map libDirPropFiles) { + // TODO: Integer value properties can be evaluated if 'simple' arithemetic + // TODO: A list of ports can be defined using keyword 'list', e.g. list(httpPort) -> 89,9889 versus literal '89,9889' + public static String getPropertyValue(String propertyName, Properties prop, Properties defaultProps, Map libertyDirPropFiles) { String value = null; - if(!libDirPropFiles.containsKey(propertyName)) { - value = props.getProperty(propertyName); - if (value == null) { - // Check for default value since no other value found. - value = defaultProps.getProperty(propertyName); - } - if (value == null && propertyName.startsWith("env.") && propertyName.length() > 4) { - // Look for property without the 'env.' prefix - String newPropName = propertyName.substring(4); - value = props.getProperty(newPropName); - if (value == null) { - // Check for default value since no other value found. - value = defaultProps.getProperty(newPropName); - } - } - } else { - File envDirectory = libDirPropFiles.get(propertyName); - value = envDirectory.toString(); - } - - if (value != null && value.startsWith("\"") && value.endsWith("\"")) { - // need to remove beginning/ending quotes - if (value.length() > 2) { - value = value.substring(1, value.length() -1); + if (libertyDirPropFiles.containsKey(propertyName)) { + return stripQuotes(libertyDirPropFiles.get(propertyName).toString()); + } + + value = lookupProperty(prop, defaultProps, propertyName); + if (value != null) { + return value; + } + + // try again with non-alphanumeric values replaced with '_', which is exactly \W in regex + String propertyNameVariation = propertyName.replaceAll("\\W", "_"); + value = lookupProperty(prop, defaultProps, propertyNameVariation); + if (value != null) { + return value; + } + + // try again with propertyNameVariation.toUpperCase() + propertyNameVariation = propertyNameVariation.toUpperCase(); + value = lookupProperty(prop, defaultProps, propertyNameVariation); + if (value != null) { + return value; + } + + // support for versions <19.0.0.3. Look for property without the 'env.' prefix + if (propertyName != null && propertyName.startsWith("env.") && propertyName.length() > 4) { + value = lookupProperty(prop, defaultProps, propertyName.substring(4)); + if (value != null) { + return value; } } + + return value; + } + + private static String stripQuotes(String value) { + if (value == null) { + return null; + } + if (value.startsWith("\"") && value.endsWith("\"") && value.length() > 2) { + return value.substring(1, value.length() - 1); + } return value; } - + + private static String lookupProperty(Properties prop, Properties defaultProps, String propertyName) { + if (prop.containsKey(propertyName)) { + return stripQuotes(prop.getProperty(propertyName)); + } + if (defaultProps.containsKey(propertyName)) { + return stripQuotes(defaultProps.getProperty(propertyName)); + } + return null; + } } diff --git a/src/test/java/io/openliberty/tools/common/config/ServerConfigDocumentOverridesTest.java b/src/test/java/io/openliberty/tools/common/config/ServerConfigDocumentOverridesTest.java new file mode 100644 index 00000000..cf283287 --- /dev/null +++ b/src/test/java/io/openliberty/tools/common/config/ServerConfigDocumentOverridesTest.java @@ -0,0 +1,237 @@ +package io.openliberty.tools.common.config; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import javax.xml.xpath.XPathExpressionException; + +import org.junit.Test; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +import io.openliberty.tools.common.TestLogger; +import io.openliberty.tools.common.plugins.config.ServerConfigDocument; +import io.openliberty.tools.common.plugins.util.ServerFeatureUtil; +import io.openliberty.tools.common.plugins.util.VariableUtility; + + +// Docs: https://openliberty.io/docs/latest/reference/config/server-configuration-overview.html +public class ServerConfigDocumentOverridesTest { + private final static Path RESOURCES_DIR = Paths.get("src/test/resources/"); + private final static Path WLP_DIR = RESOURCES_DIR.resolve("serverConfig/liberty/wlp/"); + private final static Path WLP_USER_DIR = RESOURCES_DIR.resolve("serverConfig/liberty/wlp/usr/"); + private final static Path SERVER_CONFIG_DIR = WLP_USER_DIR.resolve("servers/defaultServer"); + private final static Path SERVERS_RESOURCES_DIR = RESOURCES_DIR.resolve("servers/"); + + // 1. variable default values in server.xml file + // 6. variable values declared in the server.xml file + @Test + public void processServerXml() throws FileNotFoundException, IOException, XPathExpressionException, SAXException { + File serversResourceDir = SERVERS_RESOURCES_DIR.toFile(); + Document doc; + Map libertyDirPropMap = new HashMap(); + libertyDirPropMap.put(ServerFeatureUtil.SERVER_CONFIG_DIR, serversResourceDir); + + // no variables defined + ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger(), libertyDirPropMap, null); + File empty = new File(serversResourceDir, "emptyList.xml"); + doc = configDocument.parseDocument(empty); + configDocument.parseVariablesForBothValues(doc); + assertTrue(configDocument.getDefaultProperties().isEmpty() && configDocument.getProperties().isEmpty()); + + // variables defined + configDocument = new ServerConfigDocument(new TestLogger(), libertyDirPropMap, null); + File defined = new File(serversResourceDir, "definedVariables.xml"); + doc = configDocument.parseDocument(defined); + configDocument.parseVariablesForBothValues(doc); + assertEquals(1, configDocument.getDefaultProperties().size()); + assertEquals("9080", configDocument.getDefaultProperties().getProperty("default.http.port")); + assertEquals(1, configDocument.getProperties().size()); + assertEquals("9081", configDocument.getProperties().getProperty("http.port")); + + // variables defined in files + configDocument = new ServerConfigDocument(new TestLogger(), libertyDirPropMap, null); + File include = new File(serversResourceDir, "testIncludeParseVariables.xml"); + doc = configDocument.parseDocument(include); + assertTrue(configDocument.getDefaultProperties().isEmpty() && configDocument.getProperties().isEmpty()); + configDocument.parseIncludeVariables(doc); + assertEquals(1, configDocument.getDefaultProperties().size()); + assertEquals("9080", configDocument.getDefaultProperties().getProperty("default.http.port")); + assertEquals(1, configDocument.getProperties().size()); + assertEquals("9081", configDocument.getProperties().getProperty("http.port")); + + // server.xml configDropins precedence + File serverConfigDir = SERVER_CONFIG_DIR.toFile(); + libertyDirPropMap.put(ServerFeatureUtil.SERVER_CONFIG_DIR, serverConfigDir); + configDocument = new ServerConfigDocument(new TestLogger(), libertyDirPropMap, null); + doc = configDocument.parseDocument(new File(serverConfigDir, "server.xml")); + configDocument.processServerXml(doc); // Variable resolution warnings can be ignored here + assertEquals("1", configDocument.getProperties().getProperty("config.dropins.defaults")); + assertEquals("The value is expected to be overriden from 1 to 2", "2", configDocument.getProperties().getProperty("config.dropins.server")); + assertEquals("The value is expected to be overriden from 1 to 3", "3", configDocument.getProperties().getProperty("config.dropins.overrides")); + } + + // when a server.xml references an environment variable that could not be resolved, additionally search for: + // 1. replace all non-alphanumeric characters with underscore char '_' + // 2. change all characters to uppercase + // 3. remove the env. prefix (<19.0.0.3 behavior) + @Test + public void serverXmlEnvVarVariationLookup() throws FileNotFoundException, Exception { + File serverConfigDir = SERVER_CONFIG_DIR.toFile(); + Map libertyDirPropMap = new HashMap(); + libertyDirPropMap.put(ServerFeatureUtil.SERVER_CONFIG_DIR, serverConfigDir); + + ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger(), libertyDirPropMap, null); + Document serverXmlDoc = configDocument.parseDocument(new File(serverConfigDir, "server.xml")); + configDocument.parseVariablesForBothValues(serverXmlDoc); + assertEquals("${this.value}", configDocument.getDefaultProperties().getProperty("server.env.defined")); + assertEquals("${this.value}", configDocument.getProperties().getProperty("server.env.defined")); + assertEquals("${that.value}", configDocument.getProperties().getProperty("bootstrap.property.defined")); + + configDocument.processBootstrapProperties(); + assertFalse(configDocument.getProperties().containsKey("that.value")); + assertTrue(configDocument.getProperties().containsKey("THAT_VALUE")); + configDocument.processServerEnv(); + assertFalse(configDocument.getProperties().containsKey("this.value")); + assertTrue(configDocument.getProperties().containsKey("this_value")); + + configDocument.parseVariablesForBothValues(serverXmlDoc); + String resolveUnderscore = VariableUtility.resolveVariables(new TestLogger(), "${this.value}", + null, configDocument.getProperties(), configDocument.getDefaultProperties(), libertyDirPropMap); + assertEquals("DEFINED", resolveUnderscore); + String resolveUnderscoreToUpper = VariableUtility.resolveVariables(new TestLogger(), "${that.value}", + null, configDocument.getProperties(), configDocument.getDefaultProperties(), libertyDirPropMap); + assertEquals("DEFINED", resolveUnderscoreToUpper); + String resolveEnvPrefix = VariableUtility.resolveVariables(new TestLogger(), "${env.HOST}", + null, configDocument.getProperties(), configDocument.getDefaultProperties(), libertyDirPropMap); + assertEquals("localhost", resolveEnvPrefix); + } + + @Test + public void processServerEnv() throws FileNotFoundException, Exception { + File wlpInstallDir = WLP_DIR.toFile(); + File wlpUserDir = WLP_USER_DIR.toFile(); + File serverDir = SERVER_CONFIG_DIR.toFile(); + Map libertyDirectoryPropertyToFileMap = new HashMap(); + libertyDirectoryPropertyToFileMap.put(ServerFeatureUtil.WLP_INSTALL_DIR, wlpInstallDir); + libertyDirectoryPropertyToFileMap.put(ServerFeatureUtil.WLP_USER_DIR, wlpUserDir); + libertyDirectoryPropertyToFileMap.put(ServerFeatureUtil.SERVER_CONFIG_DIR, serverDir); + ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger(), libertyDirectoryPropertyToFileMap); + configDocument.processServerEnv(); + Properties props = configDocument.getProperties(); + + // in increasing precedence + // 1. {wlp.install.dir}/etc + assertEquals("true", props.get("etc.unique")); + + // 2. {wlp.user.dir}/shared + assertEquals("true", props.get("shared.unique")); + assertEquals("This value is expected to be overriden from false to true","true", props.get("shared.overriden")); + + // 3. {server.config.dir} + assertEquals("old_value", props.get("overriden_value")); + assertEquals("This value is expected to be overriden from 9081 to 1111", "1111", props.get("http.port")); + } + + // 3. bootstrap.properties + @Test + public void processBootstrapProperties() throws FileNotFoundException, Exception { + File serversDir = SERVERS_RESOURCES_DIR.toFile(); + ServerConfigDocument configDocument; + Map libertyDirPropMap = new HashMap(); + libertyDirPropMap.put(ServerFeatureUtil.SERVER_CONFIG_DIR, serversDir); + + // bootstrap.properties + configDocument = new ServerConfigDocument(new TestLogger(), libertyDirPropMap, null); + configDocument.processBootstrapProperties(); + assertEquals(1, configDocument.getProperties().size()); + assertEquals("extraFeatures.xml", configDocument.getProperties().getProperty("extras.filename")); + + // bootstrap.include + libertyDirPropMap.put(ServerFeatureUtil.SERVER_CONFIG_DIR, new File(serversDir, "bootstrapInclude")); + configDocument = new ServerConfigDocument(new TestLogger(), libertyDirPropMap, null); + configDocument.processBootstrapProperties(); + assertEquals(2, configDocument.getProperties().size()); + assertTrue(configDocument.getProperties().containsKey("bootstrap.include")); + assertEquals("extraFeatures.xml", configDocument.getProperties().getProperty("extras.filename")); + + // bootstrap.include termination check + libertyDirPropMap.put(ServerFeatureUtil.SERVER_CONFIG_DIR, new File(serversDir, "bootstrapOuroboros")); + configDocument = new ServerConfigDocument(new TestLogger(), libertyDirPropMap, null); + configDocument.processBootstrapProperties(); + } + + // 5. Variables loaded from files in the ${server.config.dir}/variables directory or other + // directories as specified by the VARIABLE_SOURCE_DIRS environment variable + @Test + public void variablesDir() throws FileNotFoundException, Exception { + File serversDir = SERVER_CONFIG_DIR.toFile(); + Map libertyDirPropMap = new HashMap(); + libertyDirPropMap.put(ServerFeatureUtil.SERVER_CONFIG_DIR, serversDir); + + ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger(), libertyDirPropMap); + configDocument.processVariablesDirectory(); + Properties props = configDocument.getProperties(); + assertEquals("9080", props.getProperty("httpPort")); + assertEquals("1000", props.getProperty(String.join(File.separator, "nested", "httpPort"))); + assertEquals("1", props.getProperty("VALUE_1")); + assertEquals("2", props.getProperty("VALUE_2")); + + // process VARIABLE_SOURCE_DIRS + String delimiter = (File.separator.equals("/")) ? ":" : ";"; + String variableSourceDirsTestValue = String.join(delimiter, + SERVERS_RESOURCES_DIR.resolve("variables").toString(), + SERVER_CONFIG_DIR.toString(), + "DOES_NOT_EXIST"); + Properties initProperties = new Properties(); + initProperties.put("VARIABLE_SOURCE_DIRS", variableSourceDirsTestValue); + configDocument = new ServerConfigDocument(new TestLogger(), libertyDirPropMap, initProperties); + configDocument.processVariablesDirectory(); + + props = configDocument.getProperties(); + assertEquals("outer_space", props.getProperty("outer.source")); + assertEquals("1", props.getProperty("VALUE_1")); + } + + // Run the method + @Test + public void initializeAppsLocationTest() { + File serverConfigDir = SERVER_CONFIG_DIR.toFile(); + Map libertyDirPropMap = new HashMap(); + libertyDirPropMap.put(ServerFeatureUtil.SERVER_CONFIG_DIR, serverConfigDir); + libertyDirPropMap.put(ServerFeatureUtil.WLP_INSTALL_DIR, WLP_DIR.toFile()); + libertyDirPropMap.put(ServerFeatureUtil.WLP_USER_DIR, WLP_USER_DIR.toFile()); + ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger(), libertyDirPropMap); + + Properties properties = configDocument.getProperties(); + Properties defaultProperties = configDocument.getDefaultProperties(); + + // default properties in server.xml + assertEquals(3, defaultProperties.size()); + + // server.env + // wlp/etc + assertEquals("true", properties.get("etc.unique")); + // wlp/shared > etc + assertEquals("The value is expected to be overriden from false to true","true", properties.get("shared.overriden")); + // serverConfig > shared + assertEquals("The value is expected to be overriden from 9081 to 1111","1111", properties.get("http.port")); + + // bootstrap.properties, overrides server.env + assertEquals("The value is expected to be overriden from false to true","true", properties.get("bootstrap.properties.override")); + // variables dir, overrides bootstrap.prop + assertEquals("The value is expected to be overriden from false to true","true", properties.get("variables.override")); + // configDropins, overrides variable dir + assertEquals("The value is expected to be overriden from 9080 to 7777","7777", properties.get("httpPort")); + } +} diff --git a/src/test/resources/serverConfig/liberty/wlp/etc/server.env b/src/test/resources/serverConfig/liberty/wlp/etc/server.env new file mode 100644 index 00000000..437b75ce --- /dev/null +++ b/src/test/resources/serverConfig/liberty/wlp/etc/server.env @@ -0,0 +1,3 @@ +etc.unique=true +shared.overriden=false +http.port=9080 \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/bootstrap.properties b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/bootstrap.properties index 3efd222e..883c74ad 100644 --- a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/bootstrap.properties +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/bootstrap.properties @@ -1 +1,5 @@ -extras.filename=extraFeatures.xml \ No newline at end of file +extras.filename=extraFeatures.xml +THAT_VALUE=DEFINED +bootstrap.properties.override=true +variables.override=false +HOST=localhost \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/configDropins/defaults/server.xml b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/configDropins/defaults/server.xml new file mode 100644 index 00000000..5cf6f7aa --- /dev/null +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/configDropins/defaults/server.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/configDropins/overrides/server.xml b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/configDropins/overrides/server.xml new file mode 100644 index 00000000..16445372 --- /dev/null +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/configDropins/overrides/server.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.env b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.env index 2b0a48a6..9caccade 100644 --- a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.env +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.env @@ -1 +1,5 @@ -keystore_password=C7ANPlAi0MQD154BJ5ZOURn \ No newline at end of file +keystore_password=C7ANPlAi0MQD154BJ5ZOURn +http.port=1111 +overriden_value=old_value +this_value=DEFINED +bootstrap.properties.override=false \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.xml b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.xml index bc801f17..77dfb53b 100644 --- a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.xml +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.xml @@ -21,4 +21,9 @@ + + + + + \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/httpPort b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/httpPort new file mode 100644 index 00000000..04a3078b --- /dev/null +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/httpPort @@ -0,0 +1 @@ +9080 \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/httpPort b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/httpPort new file mode 100644 index 00000000..e37d32ab --- /dev/null +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/httpPort @@ -0,0 +1 @@ +1000 \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/nested.properties b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/nested.properties new file mode 100644 index 00000000..abccd5b2 --- /dev/null +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/nested.properties @@ -0,0 +1,2 @@ +VALUE_1=1 +VALUE_2=2 \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/variables.override b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/variables.override new file mode 100644 index 00000000..f32a5804 --- /dev/null +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/variables.override @@ -0,0 +1 @@ +true \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/shared/server.env b/src/test/resources/serverConfig/liberty/wlp/usr/shared/server.env new file mode 100644 index 00000000..dcbb2923 --- /dev/null +++ b/src/test/resources/serverConfig/liberty/wlp/usr/shared/server.env @@ -0,0 +1,3 @@ +shared.unique=true +shared.overriden=true +http.port=9081 \ No newline at end of file diff --git a/src/test/resources/servers/bootstrapInclude/bootstrap.properties b/src/test/resources/servers/bootstrapInclude/bootstrap.properties new file mode 100644 index 00000000..9ebd0bd6 --- /dev/null +++ b/src/test/resources/servers/bootstrapInclude/bootstrap.properties @@ -0,0 +1 @@ +bootstrap.include=../bootstrap.properties \ No newline at end of file diff --git a/src/test/resources/servers/bootstrapOuroboros/bootstrap.properties b/src/test/resources/servers/bootstrapOuroboros/bootstrap.properties new file mode 100644 index 00000000..1f510731 --- /dev/null +++ b/src/test/resources/servers/bootstrapOuroboros/bootstrap.properties @@ -0,0 +1 @@ +bootstrap.include=bootstrap.properties \ No newline at end of file diff --git a/src/test/resources/servers/definedVariables.xml b/src/test/resources/servers/definedVariables.xml new file mode 100644 index 00000000..7ee645c6 --- /dev/null +++ b/src/test/resources/servers/definedVariables.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/src/test/resources/servers/testIncludeParseVariables.xml b/src/test/resources/servers/testIncludeParseVariables.xml new file mode 100644 index 00000000..950c5351 --- /dev/null +++ b/src/test/resources/servers/testIncludeParseVariables.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/src/test/resources/servers/variables/outer.source b/src/test/resources/servers/variables/outer.source new file mode 100644 index 00000000..f4ed48a9 --- /dev/null +++ b/src/test/resources/servers/variables/outer.source @@ -0,0 +1 @@ +outer_space \ No newline at end of file