Add support to testclusters for older versions of elasticsearch

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -202,15 +202,22 @@ private Object dependencyNotation(Project project, ElasticsearchDistribution dis
         }
 
         String extension = distribution.getType().toString();
-        String classifier = "x86_64";
+        String classifier = ":x86_64";
         if (distribution.getType() == Type.ARCHIVE) {
             extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
-            classifier = distribution.getPlatform() + "-" + classifier;
+            if (distribution.getVersion().onOrAfter("7.0.0")) {
+                classifier = ":" + distribution.getPlatform() + "-x86_64";
+            } else {
+                classifier = "";
+            }
         } else if (distribution.getType() == Type.DEB) {
-            classifier = "amd64";
+            classifier = ":amd64";
+        }
+        String flavor = "";
+        if (distribution.getFlavor() == Flavor.OSS && distribution.getVersion().onOrAfter("6.3.0")) {
+            flavor = "-oss";
         }
-        return FAKE_IVY_GROUP + ":elasticsearch" + (distribution.getFlavor() == Flavor.OSS ? "-oss:" : ":")
-            + distribution.getVersion() + ":" + classifier + "@" + extension;
+        return FAKE_IVY_GROUP + ":elasticsearch" + flavor + ":" + distribution.getVersion() + classifier + "@" + extension;
     }
 
     private static Dependency projectDependency(Project project, String projectPath, String projectConfig) {

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -161,7 +161,7 @@ public boolean getBundledJdk() {
         return bundledJdk.getOrElse(true);
     }
 
-    public void setBundledJdk(boolean bundledJdk) {
+    public void setBundledJdk(Boolean bundledJdk) {
         this.bundledJdk.set(bundledJdk);
     }
 
@@ -197,15 +197,15 @@ public Configuration getConfiguration() {
     void finalizeValues() {
 
         if (getType() == Type.INTEG_TEST_ZIP) {
-            if (platform.isPresent()) {
+            if (platform.getOrNull() != null) {
                 throw new IllegalArgumentException(
                     "platform not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
             }
-            if (flavor.isPresent()) {
+            if (flavor.getOrNull() != null) {
                 throw new IllegalArgumentException(
-                    "flavor not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
+                    "flavor [" + flavor.get() + "] not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
             }
-            if (bundledJdk.isPresent()) {
+            if (bundledJdk.getOrNull() != null) {
                 throw new IllegalArgumentException(
                     "bundledJdk not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
             }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -254,16 +254,36 @@ public void start() {
         } else {
             nodeNames = nodes.stream().map(ElasticsearchNode::getName).map(this::safeName).collect(Collectors.joining(","));
         }
+        ElasticsearchNode firstNode = null;
         for (ElasticsearchNode node : nodes) {
+            // Can only configure master nodes if we have node names defined
             if (nodeNames != null) {
-                // Can only configure master nodes if we have node names defined
-                if (node.getVersion().getMajor() >= 7) {
+                if (node.getVersion().onOrAfter("7.0.0")) {
                     node.defaultConfig.put("cluster.initial_master_nodes", "[" + nodeNames + "]");
                     node.defaultConfig.put("discovery.seed_providers", "file");
                     node.defaultConfig.put("discovery.seed_hosts", "[]");
+                } else {
+                    node.defaultConfig.put("discovery.zen.master_election.wait_for_joins_timeout", "5s");
+                    if (nodes.size() > 1) {
+                        node.defaultConfig.put("discovery.zen.minimum_master_nodes", nodes.size() / 2 + 1 + "");
+                    }
+                    if (node.getVersion().onOrAfter("6.5.0")) {
+                        node.defaultConfig.put("discovery.zen.hosts_provider", "file");
+                        node.defaultConfig.put("discovery.zen.ping.unicast.hosts", "[]");
+                    } else {
+                        if (firstNode == null) {
+                            node.defaultConfig.put("discovery.zen.ping.unicast.hosts", "[]");
+                        } else {
+                            firstNode.waitForAllConditions();
+                            node.defaultConfig.put("discovery.zen.ping.unicast.hosts", "[\"" + firstNode.getTransportPortURI() + "\"]");
+                        }
+                    }
                 }
             }
             node.start();
+            if (firstNode == null) {
+                firstNode = node;
+            }
         }
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -183,25 +183,30 @@ public Version getVersion() {
     @Override
     public void setVersion(String version) {
         requireNonNull(version, "null version passed when configuring test cluster `" + this + "`");
-        String distroName = "testclusters" + path.replace(":", "-") + "-" + this.name + "-" + version + "-";
-        NamedDomainObjectContainer<ElasticsearchDistribution> container = DistributionDownloadPlugin.getContainer(project);
-        if (container.findByName(distroName) == null){
-            container.create(distroName);
-        }
-        ElasticsearchDistribution distro = container.getByName(distroName);
-        distro.setVersion(version);
-        setDistributionType(distro, testDistribution);
-        distributions.add(distro);
+        checkFrozen();
+        distributions.clear();
+        doSetVersion(version);
     }
 
     @Override
     public void setVersions(List<String> versions) {
         requireNonNull(versions, "null version list passed when configuring test cluster `" + this + "`");
-        checkFrozen();
         distributions.clear();
         for (String version : versions) {
-            setVersion(version);
+            doSetVersion(version);
+        }
+    }
+
+    private void doSetVersion(String version) {
+        String distroName = "testclusters" + path.replace(":", "-") + "-" + this.name + "-" + version + "-";
+        NamedDomainObjectContainer<ElasticsearchDistribution> container = DistributionDownloadPlugin.getContainer(project);
+        if (container.findByName(distroName) == null) {
+            container.create(distroName);
         }
+        ElasticsearchDistribution distro = container.getByName(distroName);
+        distro.setVersion(version);
+        setDistributionType(distro, testDistribution);
+        distributions.add(distro);
     }
 
     @Internal
@@ -228,6 +233,10 @@ public void setTestDistribution(TestDistribution testDistribution) {
     private void setDistributionType(ElasticsearchDistribution distribution, TestDistribution testDistribution) {
         if (testDistribution == TestDistribution.INTEG_TEST) {
             distribution.setType(ElasticsearchDistribution.Type.INTEG_TEST_ZIP);
+            // we change the underlying distribution when changing the test distribution of the cluster.
+            distribution.setFlavor(null);
+            distribution.setPlatform(null);
+            distribution.setBundledJdk(null);
         } else {
             distribution.setType(ElasticsearchDistribution.Type.ARCHIVE);
             if (testDistribution == TestDistribution.DEFAULT) {
@@ -406,6 +415,14 @@ public synchronized void start() {
             );
         }
 
+        if (getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT) {
+            LOGGER.info("emulating the {} flavor for {} by installing x-pack", testDistribution, getVersion());
+            runElaticsearchBinScript(
+                "elasticsearch-plugin",
+                "install", "--batch", "x-pack"
+            );
+        }
+
         if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
             logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files");
             runElaticsearchBinScript("elasticsearch-keystore", "create");
@@ -428,13 +445,17 @@ public synchronized void start() {
 
         copyExtraConfigFiles();
 
-        if (isSettingMissingOrTrue("xpack.security.enabled")) {
-            logToProcessStdout("Setting up " + credentials.size() + " users");
+        if (isSettingTrue("xpack.security.enabled")) {
             if (credentials.isEmpty()) {
                 user(Collections.emptyMap());
             }
+        }
+
+        if (credentials.isEmpty() == false) {
+            logToProcessStdout("Setting up " + credentials.size() + " users");
+
             credentials.forEach(paramMap -> runElaticsearchBinScript(
-                "elasticsearch-users",
+                getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users",
                 paramMap.entrySet().stream()
                     .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue()))
                     .toArray(String[]::new)
@@ -486,7 +507,7 @@ public void goToNextVersion() {
         restart();
     }
 
-    private boolean isSettingMissingOrTrue(String name) {
+    private boolean isSettingTrue(String name) {
         return Boolean.valueOf(settings.getOrDefault(name, "false").toString());
     }
 

qa/smoke-test-bwc/build.gradle

@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.testclusters.RestTestRunnerTask
+import org.elasticsearch.gradle.testclusters.TestDistribution
+
+apply plugin: 'elasticsearch.testclusters'
+apply plugin: 'elasticsearch.standalone-test'
+
+/**
+ * This projects tests that bwc versions can be started up.
+ *
+ * The goal is to test the build infrastructure that sets the test clusters up, not the versions as those won't change,
+ * but the build does have version dependent code and we want a quick way to verify that clusters still start up on al
+ * versions after making changes.
+ */
+
+tasks.register("bwcTest") {
+    description = 'Runs backwards compatibility tests.'
+    group = 'verification'
+}
+
+/**
+ * This is useful for making sure changes in master won't break when back-porting, but these
+ * WIL NOT RUN IN CI, since CI doesn't call `bwcTest`, the matrix calls the version specific bwc test
+ * but only does so for index compatible versions, so in CI the check will be spread out across version.
+ * As such keeping this list up to date is not mandatory.
+ * Note that adding unreleased versions that are not index compatible with the current branch here won't work,
+ * as the branch won't know about them ( you can't add the unreleased maintenance version )
+ */
+List<Version> additionalVersion = [
+        "6.0.0", "6.1.0", "6.2.0", "6.2.2", "6.2.3", "6.2.4",
+        "6.3.0", "6.4.0", "6.5.0", "6.6.0", "6.7.0", "6.8.0"
+].collect { Version.fromString(it)}
+
+(bwcVersions.indexCompatible + additionalVersion).each { Version bwcVersion ->
+    tasks.register("v${bwcVersion}#bwcTest")
+    // We don't use INTEG_TEST for bwc
+    (TestDistribution.values()
+            // The integ test distribution is currently not used for bwc
+            .minus(TestDistribution.INTEG_TEST)
+    ).each { TestDistribution distro ->
+        String baseName = "v${bwcVersion}-${distro}"
+
+        testClusters {
+            "${baseName}" {
+                numberOfNodes = 2
+                testDistribution = distro
+                version = bwcVersion.toString()
+                javaHome = project.file(project.ext.runtimeJavaHome)
+                if (distro == TestDistribution.DEFAULT) {
+                    setting 'xpack.security.enabled', 'false'
+                }
+            }
+        }
+
+        tasks.register("${baseName}#test", RestTestRunnerTask) {
+            useCluster testClusters."${baseName}"
+            mustRunAfter(precommit)
+            doFirst {
+                project.delete("${buildDir}/cluster/shared/repo/${baseName}")
+            }
+
+            systemProperty 'tests.cluster_version', bwcVersion.toString().minus("-SNAPSHOT")
+            systemProperty 'tests.cluster_distro', distro
+            nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",") }")
+            nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName() }")
+        }
+
+        if (project.bwc_tests_enabled) {
+            tasks.named("v${bwcVersion}#bwcTest").configure {
+                dependsOn "${baseName}#test"
+            }
+        }
+    }
+    tasks.named("bwcTest").configure {
+        dependsOn "v${bwcVersion}#bwcTest"
+    }
+}
+
+test.enabled = false

qa/smoke-test-bwc/src/test/java/org/elasticsearch/upgrades/VersionCheckIT.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+import java.util.stream.Collectors;
+
+
+public class VersionCheckIT extends ESRestTestCase {
+
+    private static final String EXPECTED_VERSION = System.getProperty("tests.cluster_version");
+    private static final String EXPECTED_DISTRO = System.getProperty("tests.cluster_distro");
+
+    @SuppressWarnings("unchecked")
+    public void testVersion() throws Exception {
+        assertNotNull("Tests expects the tests.cluster_version to be passed in", EXPECTED_VERSION);
+        assertNotNull("Tests expects the tests.cluster_distro to be passed in", EXPECTED_DISTRO);
+
+        Map<String, Object> response = entityAsMap(
+            client().performRequest(
+                new Request("GET", "/_nodes?pretty")
+            )
+        );
+        logger.info("_nodes: {}", response);
+        Map<String, Map<String, Object> > nodes = (Map<String, Map<String, Object>>) response.get("nodes");
+        for (Map.Entry<String, Map<String, Object>> entry : nodes.entrySet()) {
+            String version = entry.getValue().get("version").toString();
+            assertEquals(EXPECTED_VERSION, version);
+            logger.info("Version of {} is {}", entry.getKey(), version);
+
+            List<String> modules = ((List<Map<String, String>>) entry.getValue().get("modules")).stream()
+                .map(each -> each.get("name"))
+                .collect(Collectors.toList());
+
+            if (EXPECTED_DISTRO.equals("DEFAULT")) {
+                if (Version.fromString(EXPECTED_VERSION).before(Version.fromString("6.3.0"))) {
+                    List<String> plugins = ((List<Map<String, String>>) entry.getValue().get("plugins")).stream()
+                        .map(each -> each.get("name"))
+                        .collect(Collectors.toList());
+                    assertTrue(
+                        "Default distribution should have x-pack plugins. Is this really the \"default\" distribution ?",
+                        plugins.stream().anyMatch(plugin -> plugin.startsWith("x-pack"))
+                    );
+                } else {
+                    assertTrue(
+                        "Default distribution should have x-pack modules. Is this really the default distribution ?",
+                        modules.stream().anyMatch(module -> module.startsWith("x-pack"))
+                    );
+                }
+            } else if (EXPECTED_DISTRO.equals("OSS")) {
+                assertFalse(
+                    "The OOS distribution should not contain x-pack modules. Is this really the oss distribution ?",
+                    modules.stream().anyMatch(module -> module.startsWith("x-pack"))
+                );
+            } else if (EXPECTED_DISTRO.equals("INMTEG_TEST_ZIP")) {
+                if (modules.size() != 0) {
+                    fail("Integ test distribution should have no modules but it had: " + modules);
+                }
+            } else {
+                fail("Unknown distribution type");
+            }
+            modules.forEach((module) -> logger.info("module: {}", module));
+
+        }
+    }
+
+    @Override
+    public boolean preserveClusterUponCompletion() {
+        // Test is read only, no need to spend time cleaning up
+        return true;
+    }
+
+}