diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java index c91bf031d1cf6..64e193d4623cf 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java @@ -202,15 +202,22 @@ private Object dependencyNotation(Project project, ElasticsearchDistribution dis } String extension = distribution.getType().toString(); - String classifier = "x86_64"; + String classifier = ":x86_64"; if (distribution.getType() == Type.ARCHIVE) { extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz"; - classifier = distribution.getPlatform() + "-" + classifier; + if (distribution.getVersion().onOrAfter("7.0.0")) { + classifier = ":" + distribution.getPlatform() + "-x86_64"; + } else { + classifier = ""; + } } else if (distribution.getType() == Type.DEB) { - classifier = "amd64"; + classifier = ":amd64"; + } + String flavor = ""; + if (distribution.getFlavor() == Flavor.OSS && distribution.getVersion().onOrAfter("6.3.0")) { + flavor = "-oss"; } - return FAKE_IVY_GROUP + ":elasticsearch" + (distribution.getFlavor() == Flavor.OSS ? "-oss:" : ":") - + distribution.getVersion() + ":" + classifier + "@" + extension; + return FAKE_IVY_GROUP + ":elasticsearch" + flavor + ":" + distribution.getVersion() + classifier + "@" + extension; } private static Dependency projectDependency(Project project, String projectPath, String projectConfig) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java index 815da77a15487..22ede2d1becff 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java @@ -161,7 +161,7 @@ public boolean getBundledJdk() { return bundledJdk.getOrElse(true); } - public void setBundledJdk(boolean bundledJdk) { + public void setBundledJdk(Boolean bundledJdk) { this.bundledJdk.set(bundledJdk); } @@ -197,15 +197,15 @@ public Configuration getConfiguration() { void finalizeValues() { if (getType() == Type.INTEG_TEST_ZIP) { - if (platform.isPresent()) { + if (platform.getOrNull() != null) { throw new IllegalArgumentException( "platform not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]"); } - if (flavor.isPresent()) { + if (flavor.getOrNull() != null) { throw new IllegalArgumentException( - "flavor not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]"); + "flavor [" + flavor.get() + "] not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]"); } - if (bundledJdk.isPresent()) { + if (bundledJdk.getOrNull() != null) { throw new IllegalArgumentException( "bundledJdk not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]"); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java index 826ec1211a66a..c7c2ca6c6687b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java @@ -254,16 +254,36 @@ public void start() { } else { nodeNames = nodes.stream().map(ElasticsearchNode::getName).map(this::safeName).collect(Collectors.joining(",")); } + ElasticsearchNode firstNode = null; for (ElasticsearchNode node : nodes) { + // Can only configure master nodes if we have node names defined if (nodeNames != null) { - // Can only configure master nodes if we have node names defined - if (node.getVersion().getMajor() >= 7) { + if (node.getVersion().onOrAfter("7.0.0")) { node.defaultConfig.put("cluster.initial_master_nodes", "[" + nodeNames + "]"); node.defaultConfig.put("discovery.seed_providers", "file"); node.defaultConfig.put("discovery.seed_hosts", "[]"); + } else { + node.defaultConfig.put("discovery.zen.master_election.wait_for_joins_timeout", "5s"); + if (nodes.size() > 1) { + node.defaultConfig.put("discovery.zen.minimum_master_nodes", nodes.size() / 2 + 1 + ""); + } + if (node.getVersion().onOrAfter("6.5.0")) { + node.defaultConfig.put("discovery.zen.hosts_provider", "file"); + node.defaultConfig.put("discovery.zen.ping.unicast.hosts", "[]"); + } else { + if (firstNode == null) { + node.defaultConfig.put("discovery.zen.ping.unicast.hosts", "[]"); + } else { + firstNode.waitForAllConditions(); + node.defaultConfig.put("discovery.zen.ping.unicast.hosts", "[\"" + firstNode.getTransportPortURI() + "\"]"); + } + } } } node.start(); + if (firstNode == null) { + firstNode = node; + } } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 5cca8dc0e1911..eee765b9da186 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -183,25 +183,30 @@ public Version getVersion() { @Override public void setVersion(String version) { requireNonNull(version, "null version passed when configuring test cluster `" + this + "`"); - String distroName = "testclusters" + path.replace(":", "-") + "-" + this.name + "-" + version + "-"; - NamedDomainObjectContainer container = DistributionDownloadPlugin.getContainer(project); - if (container.findByName(distroName) == null){ - container.create(distroName); - } - ElasticsearchDistribution distro = container.getByName(distroName); - distro.setVersion(version); - setDistributionType(distro, testDistribution); - distributions.add(distro); + checkFrozen(); + distributions.clear(); + doSetVersion(version); } @Override public void setVersions(List versions) { requireNonNull(versions, "null version list passed when configuring test cluster `" + this + "`"); - checkFrozen(); distributions.clear(); for (String version : versions) { - setVersion(version); + doSetVersion(version); + } + } + + private void doSetVersion(String version) { + String distroName = "testclusters" + path.replace(":", "-") + "-" + this.name + "-" + version + "-"; + NamedDomainObjectContainer container = DistributionDownloadPlugin.getContainer(project); + if (container.findByName(distroName) == null) { + container.create(distroName); } + ElasticsearchDistribution distro = container.getByName(distroName); + distro.setVersion(version); + setDistributionType(distro, testDistribution); + distributions.add(distro); } @Internal @@ -228,6 +233,10 @@ public void setTestDistribution(TestDistribution testDistribution) { private void setDistributionType(ElasticsearchDistribution distribution, TestDistribution testDistribution) { if (testDistribution == TestDistribution.INTEG_TEST) { distribution.setType(ElasticsearchDistribution.Type.INTEG_TEST_ZIP); + // we change the underlying distribution when changing the test distribution of the cluster. + distribution.setFlavor(null); + distribution.setPlatform(null); + distribution.setBundledJdk(null); } else { distribution.setType(ElasticsearchDistribution.Type.ARCHIVE); if (testDistribution == TestDistribution.DEFAULT) { @@ -406,6 +415,14 @@ public synchronized void start() { ); } + if (getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT) { + LOGGER.info("emulating the {} flavor for {} by installing x-pack", testDistribution, getVersion()); + runElaticsearchBinScript( + "elasticsearch-plugin", + "install", "--batch", "x-pack" + ); + } + if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) { logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files"); runElaticsearchBinScript("elasticsearch-keystore", "create"); @@ -428,13 +445,17 @@ public synchronized void start() { copyExtraConfigFiles(); - if (isSettingMissingOrTrue("xpack.security.enabled")) { - logToProcessStdout("Setting up " + credentials.size() + " users"); + if (isSettingTrue("xpack.security.enabled")) { if (credentials.isEmpty()) { user(Collections.emptyMap()); } + } + + if (credentials.isEmpty() == false) { + logToProcessStdout("Setting up " + credentials.size() + " users"); + credentials.forEach(paramMap -> runElaticsearchBinScript( - "elasticsearch-users", + getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users", paramMap.entrySet().stream() .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue())) .toArray(String[]::new) @@ -486,7 +507,7 @@ public void goToNextVersion() { restart(); } - private boolean isSettingMissingOrTrue(String name) { + private boolean isSettingTrue(String name) { return Boolean.valueOf(settings.getOrDefault(name, "false").toString()); } diff --git a/qa/smoke-test-bwc/build.gradle b/qa/smoke-test-bwc/build.gradle new file mode 100644 index 0000000000000..795633b00200e --- /dev/null +++ b/qa/smoke-test-bwc/build.gradle @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.testclusters.RestTestRunnerTask +import org.elasticsearch.gradle.testclusters.TestDistribution + +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-test' + +/** + * This projects tests that bwc versions can be started up. + * + * The goal is to test the build infrastructure that sets the test clusters up, not the versions as those won't change, + * but the build does have version dependent code and we want a quick way to verify that clusters still start up on al + * versions after making changes. + */ + +tasks.register("bwcTest") { + description = 'Runs backwards compatibility tests.' + group = 'verification' +} + +/** + * This is useful for making sure changes in master won't break when back-porting, but these + * WIL NOT RUN IN CI, since CI doesn't call `bwcTest`, the matrix calls the version specific bwc test + * but only does so for index compatible versions, so in CI the check will be spread out across version. + * As such keeping this list up to date is not mandatory. + * Note that adding unreleased versions that are not index compatible with the current branch here won't work, + * as the branch won't know about them ( you can't add the unreleased maintenance version ) + */ +List additionalVersion = [ + "6.0.0", "6.1.0", "6.2.0", "6.2.2", "6.2.3", "6.2.4", + "6.3.0", "6.4.0", "6.5.0", "6.6.0", "6.7.0", "6.8.0" +].collect { Version.fromString(it)} + +(bwcVersions.indexCompatible + additionalVersion).each { Version bwcVersion -> + tasks.register("v${bwcVersion}#bwcTest") + // We don't use INTEG_TEST for bwc + (TestDistribution.values() + // The integ test distribution is currently not used for bwc + .minus(TestDistribution.INTEG_TEST) + ).each { TestDistribution distro -> + String baseName = "v${bwcVersion}-${distro}" + + testClusters { + "${baseName}" { + numberOfNodes = 2 + testDistribution = distro + version = bwcVersion.toString() + javaHome = project.file(project.ext.runtimeJavaHome) + if (distro == TestDistribution.DEFAULT) { + setting 'xpack.security.enabled', 'false' + } + } + } + + tasks.register("${baseName}#test", RestTestRunnerTask) { + useCluster testClusters."${baseName}" + mustRunAfter(precommit) + doFirst { + project.delete("${buildDir}/cluster/shared/repo/${baseName}") + } + + systemProperty 'tests.cluster_version', bwcVersion.toString().minus("-SNAPSHOT") + systemProperty 'tests.cluster_distro', distro + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",") }") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName() }") + } + + if (project.bwc_tests_enabled) { + tasks.named("v${bwcVersion}#bwcTest").configure { + dependsOn "${baseName}#test" + } + } + } + tasks.named("bwcTest").configure { + dependsOn "v${bwcVersion}#bwcTest" + } +} + +test.enabled = false \ No newline at end of file diff --git a/qa/smoke-test-bwc/src/test/java/org/elasticsearch/upgrades/VersionCheckIT.java b/qa/smoke-test-bwc/src/test/java/org/elasticsearch/upgrades/VersionCheckIT.java new file mode 100644 index 0000000000000..fd7d09a4ab277 --- /dev/null +++ b/qa/smoke-test-bwc/src/test/java/org/elasticsearch/upgrades/VersionCheckIT.java @@ -0,0 +1,97 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.upgrades; + +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; + +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.util.List; +import java.util.Map; + +import java.util.stream.Collectors; + + +public class VersionCheckIT extends ESRestTestCase { + + private static final String EXPECTED_VERSION = System.getProperty("tests.cluster_version"); + private static final String EXPECTED_DISTRO = System.getProperty("tests.cluster_distro"); + + @SuppressWarnings("unchecked") + public void testVersion() throws Exception { + assertNotNull("Tests expects the tests.cluster_version to be passed in", EXPECTED_VERSION); + assertNotNull("Tests expects the tests.cluster_distro to be passed in", EXPECTED_DISTRO); + + Map response = entityAsMap( + client().performRequest( + new Request("GET", "/_nodes?pretty") + ) + ); + logger.info("_nodes: {}", response); + Map > nodes = (Map>) response.get("nodes"); + for (Map.Entry> entry : nodes.entrySet()) { + String version = entry.getValue().get("version").toString(); + assertEquals(EXPECTED_VERSION, version); + logger.info("Version of {} is {}", entry.getKey(), version); + + List modules = ((List>) entry.getValue().get("modules")).stream() + .map(each -> each.get("name")) + .collect(Collectors.toList()); + + if (EXPECTED_DISTRO.equals("DEFAULT")) { + if (Version.fromString(EXPECTED_VERSION).before(Version.fromString("6.3.0"))) { + List plugins = ((List>) entry.getValue().get("plugins")).stream() + .map(each -> each.get("name")) + .collect(Collectors.toList()); + assertTrue( + "Default distribution should have x-pack plugins. Is this really the \"default\" distribution ?", + plugins.stream().anyMatch(plugin -> plugin.startsWith("x-pack")) + ); + } else { + assertTrue( + "Default distribution should have x-pack modules. Is this really the default distribution ?", + modules.stream().anyMatch(module -> module.startsWith("x-pack")) + ); + } + } else if (EXPECTED_DISTRO.equals("OSS")) { + assertFalse( + "The OOS distribution should not contain x-pack modules. Is this really the oss distribution ?", + modules.stream().anyMatch(module -> module.startsWith("x-pack")) + ); + } else if (EXPECTED_DISTRO.equals("INMTEG_TEST_ZIP")) { + if (modules.size() != 0) { + fail("Integ test distribution should have no modules but it had: " + modules); + } + } else { + fail("Unknown distribution type"); + } + modules.forEach((module) -> logger.info("module: {}", module)); + + } + } + + @Override + public boolean preserveClusterUponCompletion() { + // Test is read only, no need to spend time cleaning up + return true; + } + +}