diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/ExtensionsProperties.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/ExtensionsProperties.java new file mode 100644 index 0000000000000..10a79e8325d0b --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/ExtensionsProperties.java @@ -0,0 +1,93 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle.testclusters; + +public class ExtensionsProperties { + private String name; + private String uniqueId; + private String hostAddress; + private String port; + private String version; + private String opensearchVersion; + private String minimumCompatibleVersion; + + public ExtensionsProperties( + String name, + String uniqueId, + String hostAddress, + String port, + String version, + String opensearchVersion, + String minimumCompatibleVersion + ) { + this.name = name; + this.uniqueId = uniqueId; + this.hostAddress = hostAddress; + this.port = port; + this.version = version; + this.opensearchVersion = opensearchVersion; + this.minimumCompatibleVersion = minimumCompatibleVersion; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUniqueId() { + return uniqueId; + } + + public void setUniqueId(String uniqueId) { + this.uniqueId = uniqueId; + } + + public String getHostAddress() { + return hostAddress; + } + + public void setHostAddress(String hostAddress) { + this.hostAddress = hostAddress; + } + + public String getPort() { + return port; + } + + public void setPort(String port) { + this.port = port; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getOpensearchVersion() { + return opensearchVersion; + } + + public void setOpensearchVersion(String opensearchVersion) { + this.opensearchVersion = opensearchVersion; + } + + public String getMinimumCompatibleVersion() { + return minimumCompatibleVersion; + } + + public void setMinimumCompatibleVersion(String minimumCompatibleVersion) { + this.minimumCompatibleVersion = minimumCompatibleVersion; + } +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java index ef52adab6377a..f8f921f3baceb 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java @@ -180,6 +180,11 @@ public void setTestDistribution(TestDistribution distribution) { nodes.all(each -> each.setTestDistribution(distribution)); } + @Override + public void extension(ExtensionsProperties extension) { + nodes.all(each -> each.extension(extension)); + } + @Override public void plugin(Provider plugin) { nodes.all(each -> each.plugin(plugin)); diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java index b051c15e81d6d..b8696f4e6c9f3 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java @@ -46,6 +46,8 @@ import org.opensearch.gradle.Version; import org.opensearch.gradle.VersionProperties; import org.opensearch.gradle.info.BuildParams; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; import org.gradle.api.Action; import org.gradle.api.Named; import org.gradle.api.NamedDomainObjectContainer; @@ -92,6 +94,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Scanner; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -142,6 +145,7 @@ public class OpenSearchNode implements TestClusterConfiguration { private final Map pluginAndModuleConfigurations = new HashMap<>(); private final List> plugins = new ArrayList<>(); private final List> modules = new ArrayList<>(); + private final List extensions = new ArrayList<>(); final LazyPropertyMap settings = new LazyPropertyMap<>("Settings", this); private final LazyPropertyMap keystoreSettings = new LazyPropertyMap<>("Keystore", this); private final LazyPropertyMap keystoreFiles = new LazyPropertyMap<>("Keystore files", this, FileEntry::new); @@ -432,6 +436,42 @@ public void module(String moduleProjectPath) { module(maybeCreatePluginOrModuleDependency(moduleProjectPath)); } + @Override + public void extension(ExtensionsProperties extensions) { + this.extensions.add(extensions); + } + + public void writeExtensionFiles() { + try { + // Creates extensions.yml in the target directory + Path destination = getDistroDir().resolve("extensions").resolve("extensions.yml"); + if (!Files.exists(getDistroDir().resolve("extensions"))) { + Files.createDirectory(getDistroDir().resolve("extensions")); + } + DumperOptions dumperOptions = new DumperOptions(); + TestExtensionsList extensionsList = new TestExtensionsList(this.extensions); + dumperOptions.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + Yaml yaml = new Yaml(dumperOptions); + Files.write(destination, yaml.dump(extensionsList).getBytes()); + + /* + * SnakeYaml creates a Yaml file with an unnecessary line at the top with the class name + * This section of code removes that line while keeping everything else the same. + */ + + Scanner scanner = new Scanner(destination); + scanner.nextLine(); + StringBuilder extensionsString = new StringBuilder(); + while (scanner.hasNextLine()) { + extensionsString.append("\n" + scanner.nextLine()); + } + Files.write(destination, extensionsString.toString().getBytes()); + + } catch (IOException e) { + throw new UncheckedIOException("Failed to write to extensions.yml", e); + } + } + @Override public void keystore(String key, String value) { keystoreSettings.put(key, value); @@ -608,7 +648,12 @@ public synchronized void start() { } } + if (!extensions.isEmpty()) { + writeExtensionFiles(); + } + logToProcessStdout("Creating " + currentConfig.command + " keystore with password set to [" + keystorePassword + "]"); + if (keystorePassword.length() > 0) { runOpenSearchBinScriptWithInput(keystorePassword + "\n" + keystorePassword, currentConfig.keystoreTool, "create", "-p"); } else { @@ -892,6 +937,10 @@ private void startOpenSearchProcess() { environment.clear(); environment.putAll(getOpenSearchEnvironment()); + if (!extensions.isEmpty()) { + environment.put("OPENSEARCH_JAVA_OPTS", "-Dopensearch.experimental.feature.extensions.enabled=true"); + } + // don't buffer all in memory, make sure we don't block on the default pipes processBuilder.redirectError(ProcessBuilder.Redirect.appendTo(currentConfig.stderrFile.toFile())); processBuilder.redirectOutput(ProcessBuilder.Redirect.appendTo(currentConfig.stdoutFile.toFile())); diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java index b27f205291269..7378804d59d41 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java @@ -55,6 +55,8 @@ public interface TestClusterConfiguration { void setTestDistribution(TestDistribution distribution); + void extension(ExtensionsProperties extension); + void plugin(Provider plugin); void plugin(String pluginProjectPath); diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestExtensionsList.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestExtensionsList.java new file mode 100644 index 0000000000000..ea601e26672a2 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestExtensionsList.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle.testclusters; + +import java.util.List; + +public class TestExtensionsList { + private List extensions; + + public TestExtensionsList(List extensionsList) { + extensions = extensionsList; + } + + public List getExtensions() { + return extensions; + } + + public void setExtensions(List extensionsList) { + extensions = extensionsList; + } +}