From d2f75e75f1ffa0420fb77d05eb28539709225098 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 1 Feb 2017 21:35:24 -0600 Subject: [PATCH 1/9] Work on building up keystore kv --- .../elasticsearch/gradle/test/ClusterConfiguration.groovy | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index 8d65f8c0d60fe..066e75a72682b 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -125,6 +125,8 @@ class ClusterConfiguration { Map settings = new HashMap<>() + Map keyStoreKVs = new HashMap<>() + // map from destination path, to source file Map extraConfigFiles = new HashMap<>() @@ -134,6 +136,7 @@ class ClusterConfiguration { LinkedHashMap setupCommands = new LinkedHashMap<>() + @Input void systemProperty(String property, String value) { systemProperties.put(property, value) @@ -144,6 +147,11 @@ class ClusterConfiguration { settings.put(name, value) } + @Input + void keyStoreKV(String name, Object value) { + keyStoreKVs.put(name, value) + } + @Input void plugin(String path) { Project pluginProject = project.project(path) From 1a767bf4c9bc4a3d1bd22cc3e36f4490205529f4 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 2 Feb 2017 17:18:55 -0600 Subject: [PATCH 2/9] Create keystore on build --- .../gradle/test/ClusterFormationTasks.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 3fc622ef5aa4f..a11a4f6323979 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -164,6 +164,7 @@ class ClusterFormationTasks { setup = configureStopTask(taskName(task, node, 'stopPrevious'), project, setup, node) setup = configureExtractTask(taskName(task, node, 'extract'), project, setup, node, configuration) setup = configureWriteConfigTask(taskName(task, node, 'configure'), project, setup, node, seedNode) + setup = configureCreateKeyStoreTask(taskName(task, node, 'keystore'), project, setup, node, seedNode) if (node.config.plugins.isEmpty() == false) { if (node.nodeVersion == VersionProperties.elasticsearch) { setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node) @@ -306,6 +307,15 @@ class ClusterFormationTasks { } } + /** Adds a task to create keystore */ + static Task configureCreateKeyStoreTask(String name, Project project, Task setup, NodeInfo node, NodeInfo seedNode) { + Map kvs = node.config.keyStoreKVs + Task createKeyStore = project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { + File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() + commandLine esKeyStoreUtil, 'create' + } + } + static Task configureExtraConfigFilesTask(String name, Project project, Task setup, NodeInfo node) { if (node.config.extraConfigFiles.isEmpty()) { return setup From ebca91f31773dfa27a971cb4c550b4214596d120 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 2 Feb 2017 18:44:39 -0600 Subject: [PATCH 3/9] Implement basic adding of keys --- .../gradle/test/ClusterFormationTasks.groovy | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index a11a4f6323979..b9f4da547a1f3 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -38,6 +38,8 @@ import org.gradle.api.tasks.Copy import org.gradle.api.tasks.Delete import org.gradle.api.tasks.Exec +import java.nio.charset.Charset +import java.nio.charset.StandardCharsets import java.nio.file.Paths import java.util.concurrent.TimeUnit @@ -164,7 +166,8 @@ class ClusterFormationTasks { setup = configureStopTask(taskName(task, node, 'stopPrevious'), project, setup, node) setup = configureExtractTask(taskName(task, node, 'extract'), project, setup, node, configuration) setup = configureWriteConfigTask(taskName(task, node, 'configure'), project, setup, node, seedNode) - setup = configureCreateKeyStoreTask(taskName(task, node, 'keystore'), project, setup, node, seedNode) + setup = configureCreateKeyStoreTask(taskName(task, node, 'createKeystore'), project, setup, node) + setup = configureAddKeyStoreTask(taskName(task, node, 'addToKeyStore'), project, setup, node) if (node.config.plugins.isEmpty() == false) { if (node.nodeVersion == VersionProperties.elasticsearch) { setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node) @@ -308,14 +311,25 @@ class ClusterFormationTasks { } /** Adds a task to create keystore */ - static Task configureCreateKeyStoreTask(String name, Project project, Task setup, NodeInfo node, NodeInfo seedNode) { - Map kvs = node.config.keyStoreKVs + static Task configureCreateKeyStoreTask(String name, Project project, Task setup, NodeInfo node) { + File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() Task createKeyStore = project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { - File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() commandLine esKeyStoreUtil, 'create' } } + /** Adds a task to add to keystore */ + static Task configureAddKeyStoreTask(String name, Project project, Task setup, NodeInfo node) { + Map kvs = node.config.keyStoreKVs + File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() + Task createKeyStore = project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { + commandLine esKeyStoreUtil, 'add', 'key', '-x' + } + createKeyStore.doFirst { + standardInput = new ByteArrayInputStream("value".getBytes(StandardCharsets.UTF_8)) + } + } + static Task configureExtraConfigFilesTask(String name, Project project, Task setup, NodeInfo node) { if (node.config.extraConfigFiles.isEmpty()) { return setup From 88a10e9465d3434df425b6044e0cd4d9dab60717 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 2 Feb 2017 19:26:07 -0600 Subject: [PATCH 4/9] Add multiple keys --- .../gradle/test/ClusterFormationTasks.groovy | 58 +++++++++++-------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index b9f4da547a1f3..c78ee9f65d149 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -167,7 +167,7 @@ class ClusterFormationTasks { setup = configureExtractTask(taskName(task, node, 'extract'), project, setup, node, configuration) setup = configureWriteConfigTask(taskName(task, node, 'configure'), project, setup, node, seedNode) setup = configureCreateKeyStoreTask(taskName(task, node, 'createKeystore'), project, setup, node) - setup = configureAddKeyStoreTask(taskName(task, node, 'addToKeyStore'), project, setup, node) + setup = configureAddKeyStoreTask(task, project, setup, node) if (node.config.plugins.isEmpty() == false) { if (node.nodeVersion == VersionProperties.elasticsearch) { setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node) @@ -242,12 +242,12 @@ class ClusterFormationTasks { File rpmDatabase = new File(node.baseDir, 'rpm-database') File rpmExtracted = new File(node.baseDir, 'rpm-extracted') /* Delay reading the location of the rpm file until task execution */ - Object rpm = "${ -> configuration.singleFile}" + Object rpm = "${-> configuration.singleFile}" extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) { commandLine 'rpm', '--badreloc', '--nodeps', '--noscripts', '--notriggers', - '--dbpath', rpmDatabase, - '--relocate', "/=${rpmExtracted}", - '-i', rpm + '--dbpath', rpmDatabase, + '--relocate', "/=${rpmExtracted}", + '-i', rpm doFirst { rpmDatabase.deleteDir() rpmExtracted.deleteDir() @@ -257,7 +257,7 @@ class ClusterFormationTasks { case 'deb': /* Delay reading the location of the deb file until task execution */ File debExtracted = new File(node.baseDir, 'deb-extracted') - Object deb = "${ -> configuration.singleFile}" + Object deb = "${-> configuration.singleFile}" extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) { commandLine 'dpkg-deb', '-x', deb, debExtracted doFirst { @@ -274,13 +274,13 @@ class ClusterFormationTasks { /** Adds a task to write elasticsearch.yml for the given node configuration */ static Task configureWriteConfigTask(String name, Project project, Task setup, NodeInfo node, NodeInfo seedNode) { Map esConfig = [ - 'cluster.name' : node.clusterName, - 'node.name' : "node-" + node.nodeNum, - 'pidfile' : node.pidFile, - 'path.repo' : "${node.sharedDir}/repo", - 'path.shared_data' : "${node.sharedDir}/", + 'cluster.name' : node.clusterName, + 'node.name' : "node-" + node.nodeNum, + 'pidfile' : node.pidFile, + 'path.repo' : "${node.sharedDir}/repo", + 'path.shared_data' : "${node.sharedDir}/", // Define a node attribute so we can test that it exists - 'node.attr.testattr' : 'test' + 'node.attr.testattr': 'test' ] // we set min master nodes to the total number of nodes in the cluster and // basically skip initial state recovery to allow the cluster to form using a realistic master election @@ -292,7 +292,7 @@ class ClusterFormationTasks { } esConfig['node.max_local_storage_nodes'] = node.config.numNodes esConfig['http.port'] = node.config.httpPort - esConfig['transport.tcp.port'] = node.config.transportPort + esConfig['transport.tcp.port'] = node.config.transportPort // Default the watermarks to absurdly low to prevent the tests from failing on nodes without enough disk space esConfig['cluster.routing.allocation.disk.watermark.low'] = '1b' esConfig['cluster.routing.allocation.disk.watermark.high'] = '1b' @@ -312,22 +312,32 @@ class ClusterFormationTasks { /** Adds a task to create keystore */ static Task configureCreateKeyStoreTask(String name, Project project, Task setup, NodeInfo node) { - File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() + File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() Task createKeyStore = project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { commandLine esKeyStoreUtil, 'create' } } /** Adds a task to add to keystore */ - static Task configureAddKeyStoreTask(String name, Project project, Task setup, NodeInfo node) { + static Task configureAddKeyStoreTask(Task parent, Project project, Task setup, NodeInfo node) { Map kvs = node.config.keyStoreKVs - File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() - Task createKeyStore = project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { - commandLine esKeyStoreUtil, 'add', 'key', '-x' - } - createKeyStore.doFirst { - standardInput = new ByteArrayInputStream("value".getBytes(StandardCharsets.UTF_8)) + File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() + List ks = ["l", "m", "n"] +// List tasks = [] +// Task createKeyStore = project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { + Task t1 = setup + for (String k in ks) { + Task t = project.tasks.create(name: taskName(parent, node, 'addToKeyStore' + k), type: LoggedExec, dependsOn: t1) { + commandLine esKeyStoreUtil, 'add', k, '-x' + } + t.doFirst { + standardInput = new ByteArrayInputStream("value".getBytes(StandardCharsets.UTF_8)) + } + t1 = t; +// tasks.add(t) +// } } + return t1 } static Task configureExtraConfigFilesTask(String name, Project project, Task setup, NodeInfo node) { @@ -337,7 +347,7 @@ class ClusterFormationTasks { Copy copyConfig = project.tasks.create(name: name, type: Copy, dependsOn: setup) File configDir = new File(node.homeDir, 'config') copyConfig.into(configDir) // copy must always have a general dest dir, even though we don't use it - for (Map.Entry extraConfigFile : node.config.extraConfigFiles.entrySet()) { + for (Map.Entry extraConfigFile : node.config.extraConfigFiles.entrySet()) { Object extraConfigFileValue = extraConfigFile.getValue() copyConfig.doFirst { // make sure the copy won't be a no-op or act on a directory @@ -617,7 +627,7 @@ class ClusterFormationTasks { return project.tasks.create(name: name, type: Exec, dependsOn: depends) { onlyIf { node.pidFile.exists() } // the pid file won't actually be read until execution time, since the read is wrapped within an inner closure of the GString - ext.pid = "${ -> node.pidFile.getText('UTF-8').trim()}" + ext.pid = "${-> node.pidFile.getText('UTF-8').trim()}" File jps if (Os.isFamily(Os.FAMILY_WINDOWS)) { jps = getJpsExecutableByName(project, "jps.exe") @@ -654,7 +664,7 @@ class ClusterFormationTasks { return project.tasks.create(name: name, type: LoggedExec, dependsOn: depends) { onlyIf { node.pidFile.exists() } // the pid file won't actually be read until execution time, since the read is wrapped within an inner closure of the GString - ext.pid = "${ -> node.pidFile.getText('UTF-8').trim()}" + ext.pid = "${-> node.pidFile.getText('UTF-8').trim()}" doFirst { logger.info("Shutting down external node with pid ${pid}") } From 1174ab9bac7334dcbbd08c36080fe9cea7f14f7d Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 3 Feb 2017 11:42:22 -0600 Subject: [PATCH 5/9] Add key values from build files --- .../gradle/test/ClusterConfiguration.groovy | 4 +-- .../gradle/test/ClusterFormationTasks.groovy | 27 ++++++++----------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index 066e75a72682b..f19a577bdd8c8 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -125,7 +125,7 @@ class ClusterConfiguration { Map settings = new HashMap<>() - Map keyStoreKVs = new HashMap<>() + Map keyStoreKVs = new HashMap<>() // map from destination path, to source file Map extraConfigFiles = new HashMap<>() @@ -148,7 +148,7 @@ class ClusterConfiguration { } @Input - void keyStoreKV(String name, Object value) { + void keyStoreKV(String name, String value) { keyStoreKVs.put(name, value) } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index c78ee9f65d149..40fda22c53936 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -38,7 +38,6 @@ import org.gradle.api.tasks.Copy import org.gradle.api.tasks.Delete import org.gradle.api.tasks.Exec -import java.nio.charset.Charset import java.nio.charset.StandardCharsets import java.nio.file.Paths import java.util.concurrent.TimeUnit @@ -166,8 +165,8 @@ class ClusterFormationTasks { setup = configureStopTask(taskName(task, node, 'stopPrevious'), project, setup, node) setup = configureExtractTask(taskName(task, node, 'extract'), project, setup, node, configuration) setup = configureWriteConfigTask(taskName(task, node, 'configure'), project, setup, node, seedNode) - setup = configureCreateKeyStoreTask(taskName(task, node, 'createKeystore'), project, setup, node) - setup = configureAddKeyStoreTask(task, project, setup, node) + setup = configureCreateKeyStoreTask(taskName(task, node, 'createKeyStore'), project, setup, node) + setup = configureAddKeyStoreTasks(task, project, setup, node) if (node.config.plugins.isEmpty() == false) { if (node.nodeVersion == VersionProperties.elasticsearch) { setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node) @@ -319,25 +318,21 @@ class ClusterFormationTasks { } /** Adds a task to add to keystore */ - static Task configureAddKeyStoreTask(Task parent, Project project, Task setup, NodeInfo node) { + static Task configureAddKeyStoreTasks(Task parent, Project project, Task setup, NodeInfo node) { Map kvs = node.config.keyStoreKVs File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() - List ks = ["l", "m", "n"] -// List tasks = [] -// Task createKeyStore = project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { - Task t1 = setup - for (String k in ks) { - Task t = project.tasks.create(name: taskName(parent, node, 'addToKeyStore' + k), type: LoggedExec, dependsOn: t1) { - commandLine esKeyStoreUtil, 'add', k, '-x' + Task parentTask = setup + for (Map.Entry entry in kvs) { + String key = entry.getKey() + Task t = project.tasks.create(name: taskName(parent, node, 'addToKeyStore#' + key), type: LoggedExec, dependsOn: parentTask) { + commandLine esKeyStoreUtil, 'add', key, '-x' } t.doFirst { - standardInput = new ByteArrayInputStream("value".getBytes(StandardCharsets.UTF_8)) + standardInput = new ByteArrayInputStream(entry.getValue().getBytes(StandardCharsets.UTF_8)) } - t1 = t; -// tasks.add(t) -// } + parentTask = t } - return t1 + return parentTask } static Task configureExtraConfigFilesTask(String name, Project project, Task setup, NodeInfo node) { From 7dd4d7efc39e3f0ce678ece912331a6d1107228d Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 3 Feb 2017 11:58:56 -0600 Subject: [PATCH 6/9] Change name of setting --- .../elasticsearch/gradle/test/ClusterConfiguration.groovy | 6 +++--- .../elasticsearch/gradle/test/ClusterFormationTasks.groovy | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index f19a577bdd8c8..4274ef32d6ad4 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -125,7 +125,7 @@ class ClusterConfiguration { Map settings = new HashMap<>() - Map keyStoreKVs = new HashMap<>() + Map keyStoreSetting = new HashMap<>() // map from destination path, to source file Map extraConfigFiles = new HashMap<>() @@ -148,8 +148,8 @@ class ClusterConfiguration { } @Input - void keyStoreKV(String name, String value) { - keyStoreKVs.put(name, value) + void keyStoreSetting(String name, String value) { + keyStoreSetting.put(name, value) } @Input diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 40fda22c53936..385050d41ec1e 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -319,7 +319,7 @@ class ClusterFormationTasks { /** Adds a task to add to keystore */ static Task configureAddKeyStoreTasks(Task parent, Project project, Task setup, NodeInfo node) { - Map kvs = node.config.keyStoreKVs + Map kvs = node.config.keyStoreSetting File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() Task parentTask = setup for (Map.Entry entry in kvs) { From 27ed1f8f8aae0c3c700a1f77a87d43faf5ff4d3a Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 3 Feb 2017 12:10:10 -0600 Subject: [PATCH 7/9] revert formatting --- .../gradle/test/ClusterFormationTasks.groovy | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 385050d41ec1e..bb869b1e728c6 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -241,12 +241,12 @@ class ClusterFormationTasks { File rpmDatabase = new File(node.baseDir, 'rpm-database') File rpmExtracted = new File(node.baseDir, 'rpm-extracted') /* Delay reading the location of the rpm file until task execution */ - Object rpm = "${-> configuration.singleFile}" + Object rpm = "${ -> configuration.singleFile}" extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) { commandLine 'rpm', '--badreloc', '--nodeps', '--noscripts', '--notriggers', - '--dbpath', rpmDatabase, - '--relocate', "/=${rpmExtracted}", - '-i', rpm + '--dbpath', rpmDatabase, + '--relocate', "/=${rpmExtracted}", + '-i', rpm doFirst { rpmDatabase.deleteDir() rpmExtracted.deleteDir() @@ -256,7 +256,7 @@ class ClusterFormationTasks { case 'deb': /* Delay reading the location of the deb file until task execution */ File debExtracted = new File(node.baseDir, 'deb-extracted') - Object deb = "${-> configuration.singleFile}" + Object deb = "${ -> configuration.singleFile}" extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) { commandLine 'dpkg-deb', '-x', deb, debExtracted doFirst { @@ -273,13 +273,13 @@ class ClusterFormationTasks { /** Adds a task to write elasticsearch.yml for the given node configuration */ static Task configureWriteConfigTask(String name, Project project, Task setup, NodeInfo node, NodeInfo seedNode) { Map esConfig = [ - 'cluster.name' : node.clusterName, - 'node.name' : "node-" + node.nodeNum, - 'pidfile' : node.pidFile, - 'path.repo' : "${node.sharedDir}/repo", - 'path.shared_data' : "${node.sharedDir}/", + 'cluster.name' : node.clusterName, + 'node.name' : "node-" + node.nodeNum, + 'pidfile' : node.pidFile, + 'path.repo' : "${node.sharedDir}/repo", + 'path.shared_data' : "${node.sharedDir}/", // Define a node attribute so we can test that it exists - 'node.attr.testattr': 'test' + 'node.attr.testattr' : 'test' ] // we set min master nodes to the total number of nodes in the cluster and // basically skip initial state recovery to allow the cluster to form using a realistic master election @@ -291,7 +291,7 @@ class ClusterFormationTasks { } esConfig['node.max_local_storage_nodes'] = node.config.numNodes esConfig['http.port'] = node.config.httpPort - esConfig['transport.tcp.port'] = node.config.transportPort + esConfig['transport.tcp.port'] = node.config.transportPort // Default the watermarks to absurdly low to prevent the tests from failing on nodes without enough disk space esConfig['cluster.routing.allocation.disk.watermark.low'] = '1b' esConfig['cluster.routing.allocation.disk.watermark.high'] = '1b' @@ -317,7 +317,7 @@ class ClusterFormationTasks { } } - /** Adds a task to add to keystore */ + /** Adds tasks to add to keystore */ static Task configureAddKeyStoreTasks(Task parent, Project project, Task setup, NodeInfo node) { Map kvs = node.config.keyStoreSetting File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() @@ -342,7 +342,7 @@ class ClusterFormationTasks { Copy copyConfig = project.tasks.create(name: name, type: Copy, dependsOn: setup) File configDir = new File(node.homeDir, 'config') copyConfig.into(configDir) // copy must always have a general dest dir, even though we don't use it - for (Map.Entry extraConfigFile : node.config.extraConfigFiles.entrySet()) { + for (Map.Entry extraConfigFile : node.config.extraConfigFiles.entrySet()) { Object extraConfigFileValue = extraConfigFile.getValue() copyConfig.doFirst { // make sure the copy won't be a no-op or act on a directory @@ -659,7 +659,7 @@ class ClusterFormationTasks { return project.tasks.create(name: name, type: LoggedExec, dependsOn: depends) { onlyIf { node.pidFile.exists() } // the pid file won't actually be read until execution time, since the read is wrapped within an inner closure of the GString - ext.pid = "${-> node.pidFile.getText('UTF-8').trim()}" + ext.pid = "${ -> node.pidFile.getText('UTF-8').trim()}" doFirst { logger.info("Shutting down external node with pid ${pid}") } From 08c1b64104534fab22805e20b8e7dc3a4feb2569 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 3 Feb 2017 12:12:33 -0600 Subject: [PATCH 8/9] revert formatting 2 --- .../org/elasticsearch/gradle/test/ClusterConfiguration.groovy | 1 - .../org/elasticsearch/gradle/test/ClusterFormationTasks.groovy | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index 4274ef32d6ad4..b53125fbbc092 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -136,7 +136,6 @@ class ClusterConfiguration { LinkedHashMap setupCommands = new LinkedHashMap<>() - @Input void systemProperty(String property, String value) { systemProperties.put(property, value) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index bb869b1e728c6..c3c3ca19a3ddc 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -622,7 +622,7 @@ class ClusterFormationTasks { return project.tasks.create(name: name, type: Exec, dependsOn: depends) { onlyIf { node.pidFile.exists() } // the pid file won't actually be read until execution time, since the read is wrapped within an inner closure of the GString - ext.pid = "${-> node.pidFile.getText('UTF-8').trim()}" + ext.pid = "${ -> node.pidFile.getText('UTF-8').trim()}" File jps if (Os.isFamily(Os.FAMILY_WINDOWS)) { jps = getJpsExecutableByName(project, "jps.exe") From 20881b5eebf8172bd7ceb2f1efd09b89421ffb73 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 21 Feb 2017 15:14:18 -0600 Subject: [PATCH 9/9] Fix a number of issues that came up from review --- .../gradle/test/ClusterConfiguration.groovy | 6 ++--- .../gradle/test/ClusterFormationTasks.groovy | 27 ++++++++++--------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index b53125fbbc092..4a88473525983 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -125,7 +125,7 @@ class ClusterConfiguration { Map settings = new HashMap<>() - Map keyStoreSetting = new HashMap<>() + Map keystoreSettings = new HashMap<>() // map from destination path, to source file Map extraConfigFiles = new HashMap<>() @@ -147,8 +147,8 @@ class ClusterConfiguration { } @Input - void keyStoreSetting(String name, String value) { - keyStoreSetting.put(name, value) + void keystoreSetting(String name, String value) { + keystoreSettings.put(name, value) } @Input diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index c3c3ca19a3ddc..38ffd58bb533c 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -165,8 +165,8 @@ class ClusterFormationTasks { setup = configureStopTask(taskName(task, node, 'stopPrevious'), project, setup, node) setup = configureExtractTask(taskName(task, node, 'extract'), project, setup, node, configuration) setup = configureWriteConfigTask(taskName(task, node, 'configure'), project, setup, node, seedNode) - setup = configureCreateKeyStoreTask(taskName(task, node, 'createKeyStore'), project, setup, node) - setup = configureAddKeyStoreTasks(task, project, setup, node) + setup = configureCreateKeystoreTask(taskName(task, node, 'createKeystore'), project, setup, node) + setup = configureAddKeystoreSettingTasks(task, project, setup, node) if (node.config.plugins.isEmpty() == false) { if (node.nodeVersion == VersionProperties.elasticsearch) { setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node) @@ -310,23 +310,24 @@ class ClusterFormationTasks { } /** Adds a task to create keystore */ - static Task configureCreateKeyStoreTask(String name, Project project, Task setup, NodeInfo node) { - File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() - Task createKeyStore = project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { - commandLine esKeyStoreUtil, 'create' + static Task configureCreateKeystoreTask(String name, Project project, Task setup, NodeInfo node) { + if (node.config.keystoreSettings.isEmpty()) { + return setup + } else { + File esKeystoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() + return configureExecTask(name, project, setup, node, esKeystoreUtil, 'create') } } - /** Adds tasks to add to keystore */ - static Task configureAddKeyStoreTasks(Task parent, Project project, Task setup, NodeInfo node) { - Map kvs = node.config.keyStoreSetting - File esKeyStoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() + /** Adds tasks to add settings to the keystore */ + static Task configureAddKeystoreSettingTasks(Task parent, Project project, Task setup, NodeInfo node) { + Map kvs = node.config.keystoreSettings + File esKeystoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile() Task parentTask = setup for (Map.Entry entry in kvs) { String key = entry.getKey() - Task t = project.tasks.create(name: taskName(parent, node, 'addToKeyStore#' + key), type: LoggedExec, dependsOn: parentTask) { - commandLine esKeyStoreUtil, 'add', key, '-x' - } + String name = taskName(parent, node, 'addToKeystore#' + key) + Task t = configureExecTask(name, project, parentTask, node, esKeystoreUtil, 'add', key, '-x') t.doFirst { standardInput = new ByteArrayInputStream(entry.getValue().getBytes(StandardCharsets.UTF_8)) }