diff --git a/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle b/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle index b80c450c5914e..a5e74c3721297 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle @@ -9,6 +9,8 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask ext.bwcTaskName = { Version version -> return "v${version}#bwcTest" @@ -36,5 +38,17 @@ plugins.withType(ElasticsearchTestBasePlugin) { } } +plugins.withType(InternalJavaRestTestPlugin) { + tasks.named("javaRestTest") { + enabled = false + } + + tasks.withType(StandaloneRestIntegTestTask).configureEach { + testClassesDirs = sourceSets.javaRestTest.output.classesDirs + classpath = sourceSets.javaRestTest.runtimeClasspath + usesDefaultDistribution() + } +} + tasks.matching { it.name.equals("check") }.configureEach {dependsOn(bwcTestSnapshots) } tasks.matching { it.name.equals("test") }.configureEach {enabled = false} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java index 854dc6d204382..c6758092b17ec 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java @@ -89,7 +89,7 @@ public void execute(Task t) { test.getJvmArgumentProviders().add(nonInputProperties); test.getExtensions().add("nonInputProperties", nonInputProperties); - test.setWorkingDir(project.file(project.getBuildDir() + "/testrun/" + test.getName())); + test.setWorkingDir(project.file(project.getBuildDir() + "/testrun/" + test.getName().replace("#", "_"))); test.setMaxParallelForks(Integer.parseInt(System.getProperty("tests.jvms", BuildParams.getDefaultParallel().toString()))); test.exclude("**/*$*.class"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java index fae845b229651..4952085f466be 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java @@ -32,7 +32,7 @@ public void registerTestArtifactFromSourceSet(SourceSet sourceSet) { JavaPluginExtension javaPluginExtension = project.getExtensions().getByType(JavaPluginExtension.class); javaPluginExtension.registerFeature(name + "Artifacts", featureSpec -> { featureSpec.usingSourceSet(sourceSet); - featureSpec.capability("org.elasticsearch.gradle", project.getName() + "-" + name + "-artifacts", "1.0"); + featureSpec.capability("org.elasticsearch.gradle", project.getName() + "-test-artifacts", "1.0"); // This feature is only used internally in the // elasticsearch build so we do not need any publication. featureSpec.disablePublication(); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java index 9baa17bc00d7c..1a7b5bc3ee2a1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java @@ -13,6 +13,8 @@ import org.elasticsearch.gradle.Architecture; import org.elasticsearch.gradle.DistributionDownloadPlugin; import org.elasticsearch.gradle.ElasticsearchDistribution; +import org.elasticsearch.gradle.ElasticsearchDistributionType; +import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes; import org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin; @@ -58,6 +60,8 @@ public class RestTestBasePlugin implements Plugin { private static final String TESTS_RUNTIME_JAVA_SYSPROP = "tests.runtime.java"; private static final String DEFAULT_DISTRIBUTION_SYSPROP = "tests.default.distribution"; private static final String INTEG_TEST_DISTRIBUTION_SYSPROP = "tests.integ-test.distribution"; + private static final String BWC_SNAPSHOT_DISTRIBUTION_SYSPROP_PREFIX = "tests.snapshot.distribution."; + private static final String BWC_RELEASED_DISTRIBUTION_SYSPROP_PREFIX = "tests.release.distribution."; private static final String TESTS_CLUSTER_MODULES_PATH_SYSPROP = "tests.cluster.modules.path"; private static final String TESTS_CLUSTER_PLUGINS_PATH_SYSPROP = "tests.cluster.plugins.path"; private static final String DEFAULT_REST_INTEG_TEST_DISTRO = "default_distro"; @@ -79,16 +83,17 @@ public void apply(Project project) { project.getPluginManager().apply(InternalDistributionDownloadPlugin.class); // Register integ-test and default distributions - NamedDomainObjectContainer distributions = DistributionDownloadPlugin.getContainer(project); - ElasticsearchDistribution defaultDistro = distributions.create(DEFAULT_REST_INTEG_TEST_DISTRO, distro -> { - distro.setVersion(VersionProperties.getElasticsearch()); - distro.setArchitecture(Architecture.current()); - }); - ElasticsearchDistribution integTestDistro = distributions.create(INTEG_TEST_REST_INTEG_TEST_DISTRO, distro -> { - distro.setVersion(VersionProperties.getElasticsearch()); - distro.setArchitecture(Architecture.current()); - distro.setType(ElasticsearchDistributionTypes.INTEG_TEST_ZIP); - }); + ElasticsearchDistribution defaultDistro = createDistribution( + project, + DEFAULT_REST_INTEG_TEST_DISTRO, + VersionProperties.getElasticsearch() + ); + ElasticsearchDistribution integTestDistro = createDistribution( + project, + INTEG_TEST_REST_INTEG_TEST_DISTRO, + VersionProperties.getElasticsearch(), + ElasticsearchDistributionTypes.INTEG_TEST_ZIP + ); // Create configures for module and plugin dependencies Configuration modulesConfiguration = createPluginConfiguration(project, MODULES_CONFIGURATION, true, false); @@ -151,6 +156,35 @@ public Void call(Object... args) { return null; } }); + + // Add `usesBwcDistribution(version)` extension method to test tasks to indicate they require a BWC distribution + task.getExtensions().getExtraProperties().set("usesBwcDistribution", new Closure(task) { + @Override + public Void call(Object... args) { + if (args.length != 1 && args[0] instanceof Version == false) { + throw new IllegalArgumentException("Expected exactly one argument of type org.elasticsearch.gradle.Version"); + } + + Version version = (Version) args[0]; + boolean isReleased = BuildParams.getBwcVersions().unreleasedInfo(version) == null; + String versionString = version.toString(); + ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString); + + task.dependsOn(bwcDistro); + registerDistributionInputs(task, bwcDistro); + + nonInputSystemProperties.systemProperty( + (isReleased ? BWC_RELEASED_DISTRIBUTION_SYSPROP_PREFIX : BWC_SNAPSHOT_DISTRIBUTION_SYSPROP_PREFIX) + versionString, + providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath()) + ); + + if (version.before(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion())) { + // If we are upgrade testing older versions we also need to upgrade to 7.last + this.call(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion()); + } + return null; + } + }); }); project.getTasks() @@ -158,6 +192,26 @@ public Void call(Object... args) { .configure(check -> check.dependsOn(project.getTasks().withType(StandaloneRestIntegTestTask.class))); } + private ElasticsearchDistribution createDistribution(Project project, String name, String version) { + return createDistribution(project, name, version, null); + } + + private ElasticsearchDistribution createDistribution(Project project, String name, String version, ElasticsearchDistributionType type) { + NamedDomainObjectContainer distributions = DistributionDownloadPlugin.getContainer(project); + ElasticsearchDistribution maybeDistro = distributions.findByName(name); + if (maybeDistro == null) { + return distributions.create(name, distro -> { + distro.setVersion(version); + distro.setArchitecture(Architecture.current()); + if (type != null) { + distro.setType(type); + } + }); + } else { + return maybeDistro; + } + } + private FileTree getDistributionFiles(ElasticsearchDistribution distribution, Action patternFilter) { return distribution.getExtracted().getAsFileTree().matching(patternFilter); } diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle index a3af45b43363e..b6f181809e0e4 100644 --- a/qa/full-cluster-restart/build.gradle +++ b/qa/full-cluster-restart/build.gradle @@ -6,64 +6,20 @@ * Side Public License, v 1. */ - -import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask -apply plugin: 'elasticsearch.internal-testclusters' -apply plugin: 'elasticsearch.standalone-rest-test' -apply plugin: 'elasticsearch.internal-test-artifact' +apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.internal-test-artifact-base' apply plugin: 'elasticsearch.bwc-test' -BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> - def baseCluster = testClusters.register(baseName) { - if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) { - // When testing older versions we have to first upgrade to 7.last - versions = [bwcVersion.toString(), BuildParams.bwcVersions.minimumWireCompatibleVersion.toString(), project.version] - } else { - versions = [bwcVersion.toString(), project.version] - } - numberOfNodes = 2 - // some tests rely on the translog not being flushed - setting 'indices.memory.shard_inactive_time', '60m' - setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" - setting 'xpack.security.enabled', 'false' - requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") - } - - tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { - useCluster baseCluster - mustRunAfter("precommit") - doFirst { - delete("${buildDir}/cluster/shared/repo/${baseName}") - } - - systemProperty 'tests.is_old_cluster', 'true' - } - - tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) { - useCluster baseCluster - dependsOn "${baseName}#oldClusterTest" - doFirst { - baseCluster.get().goToNextVersion() - if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) { - // When doing a full cluster restart of older versions we actually have to upgrade twice. First to 7.last, then to the current version. - baseCluster.get().goToNextVersion() - } - } - systemProperty 'tests.is_old_cluster', 'false' - } - - String oldVersion = bwcVersion.toString().minus("-SNAPSHOT") - tasks.matching { it.name.startsWith(baseName) && it.name.endsWith("ClusterTest") }.configureEach { - it.systemProperty 'tests.old_cluster_version', oldVersion - it.systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}" - it.nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) - it.nonInputProperties.systemProperty('tests.clustername', baseName) - } +testArtifacts { + registerTestArtifactFromSourceSet(sourceSets.javaRestTest) +} - tasks.register(bwcTaskName(bwcVersion)) { - dependsOn tasks.named("${baseName}#upgradedClusterTest") +BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> + tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { + usesBwcDistribution(bwcVersion) + systemProperty("tests.old_cluster_version", bwcVersion) } } diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java similarity index 97% rename from qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java rename to qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index af66fbc61562b..3f9a007e6bf4e 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -8,6 +8,8 @@ package org.elasticsearch.upgrades; +import com.carrotsearch.randomizedtesting.annotations.Name; + import org.apache.http.util.EntityUtils; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.settings.RestClusterGetSettingsResponse; @@ -28,6 +30,10 @@ import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.test.NotEqualMessageBuilder; import org.elasticsearch.test.XContentTestUtils; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.transport.Compression; @@ -35,6 +41,10 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; import java.io.IOException; import java.util.ArrayList; @@ -44,7 +54,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -80,13 +89,41 @@ * version is started with the same data directories and then this is rerun * with {@code tests.is_old_cluster} set to {@code false}. */ -public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase { +public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCase { + + private static TemporaryFolder repoDirectory = new TemporaryFolder(); + + protected static LocalClusterConfigProvider clusterConfig = c -> {}; + + private static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(2) + .setting("path.repo", () -> repoDirectory.getRoot().getPath()) + .setting("xpack.security.enabled", "false") + // some tests rely on the translog not being flushed + .setting("indices.memory.shard_inactive_time", "60m") + .apply(() -> clusterConfig) + .feature(FeatureFlag.TIME_SERIES_MODE) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster); private String index; + public FullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } + @Before public void setIndex() { - index = getTestName().toLowerCase(Locale.ROOT); + index = getRootTestName(); } public void testSearch() throws Exception { @@ -1051,7 +1088,7 @@ public void testSnapshotRestore() throws IOException { repoConfig.startObject("settings"); { repoConfig.field("compress", randomBoolean()); - repoConfig.field("location", System.getProperty("tests.path.repo")); + repoConfig.field("location", repoDirectory.getRoot().getPath()); } repoConfig.endObject(); } @@ -1725,7 +1762,7 @@ public void testEnableSoftDeletesOnRestore() throws Exception { repoConfig.startObject("settings"); { repoConfig.field("compress", randomBoolean()); - repoConfig.field("location", System.getProperty("tests.path.repo")); + repoConfig.field("location", repoDirectory.getRoot().getPath()); } repoConfig.endObject(); } @@ -1785,7 +1822,7 @@ public void testForbidDisableSoftDeletesOnRestore() throws Exception { repoConfig.startObject("settings"); { repoConfig.field("compress", randomBoolean()); - repoConfig.field("location", System.getProperty("tests.path.repo")); + repoConfig.field("location", repoDirectory.getRoot().getPath()); } repoConfig.endObject(); } diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartTestOrdering.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartTestOrdering.java new file mode 100644 index 0000000000000..232619ee93bb9 --- /dev/null +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartTestOrdering.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.upgrades; + +import com.carrotsearch.randomizedtesting.TestMethodAndParams; + +import java.util.Comparator; + +public class FullClusterRestartTestOrdering implements Comparator { + @Override + public int compare(TestMethodAndParams o1, TestMethodAndParams o2) { + return Integer.compare(getOrdinal(o1), getOrdinal(o2)); + } + + private int getOrdinal(TestMethodAndParams t) { + return ((FullClusterRestartUpgradeStatus) t.getInstanceArguments().get(0)).ordinal(); + } +} diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartUpgradeStatus.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartUpgradeStatus.java new file mode 100644 index 0000000000000..06048d020e2a0 --- /dev/null +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartUpgradeStatus.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.upgrades; + +public enum FullClusterRestartUpgradeStatus { + OLD, + UPGRADED +} diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java new file mode 100644 index 0000000000000..a064c87743800 --- /dev/null +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.upgrades; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.util.Version; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.AfterClass; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Locale; + +import static org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus.OLD; +import static org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus.UPGRADED; + +@TestCaseOrdering(FullClusterRestartTestOrdering.class) +public abstract class ParameterizedFullClusterRestartTestCase extends ESRestTestCase { + private static final Version MINIMUM_WIRE_COMPATIBLE_VERSION = Version.fromString("7.17.0"); + private static final Version OLD_CLUSTER_VERSION = Version.fromString(System.getProperty("tests.old_cluster_version")); + private static boolean upgradeFailed = false; + private static boolean upgraded = false; + private final FullClusterRestartUpgradeStatus requestedUpgradeStatus; + + public ParameterizedFullClusterRestartTestCase(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + this.requestedUpgradeStatus = upgradeStatus; + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return Arrays.stream(FullClusterRestartUpgradeStatus.values()).map(v -> new Object[] { v }).toList(); + } + + @Before + public void maybeUpgrade() throws Exception { + if (upgraded == false && requestedUpgradeStatus == UPGRADED) { + try { + if (OLD_CLUSTER_VERSION.before(MINIMUM_WIRE_COMPATIBLE_VERSION)) { + // First upgrade to latest wire compatible version + getUpgradeCluster().upgradeToVersion(MINIMUM_WIRE_COMPATIBLE_VERSION); + } + getUpgradeCluster().upgradeToVersion(Version.CURRENT); + closeClients(); + initClient(); + } catch (Exception e) { + upgradeFailed = true; + throw e; + } finally { + upgraded = true; + } + } + + // Skip remaining tests if upgrade failed + assumeFalse("Cluster upgrade failed", upgradeFailed); + } + + @AfterClass + public static void resetUpgrade() { + upgraded = false; + upgradeFailed = false; + } + + public boolean isRunningAgainstOldCluster() { + return requestedUpgradeStatus == OLD; + } + + public static org.elasticsearch.Version getOldClusterVersion() { + return org.elasticsearch.Version.fromString(OLD_CLUSTER_VERSION.toString()); + } + + public static Version getOldClusterTestVersion() { + return Version.fromString(OLD_CLUSTER_VERSION.toString()); + } + + protected abstract ElasticsearchCluster getUpgradeCluster(); + + @Override + protected String getTestRestCluster() { + return getUpgradeCluster().getHttpAddresses(); + } + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + protected String getRootTestName() { + return getTestName().split(" ")[0].toLowerCase(Locale.ROOT); + } +} diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java similarity index 91% rename from qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java rename to qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java index d69f0b05958f9..1636644409fc7 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java @@ -8,6 +8,8 @@ package org.elasticsearch.upgrades; +import com.carrotsearch.randomizedtesting.annotations.Name; + import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; @@ -32,7 +34,11 @@ import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.xcontent.XContentBuilder; +import org.junit.ClassRule; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -54,10 +60,29 @@ * The queries to test are specified in json format, which turns out to work because we tend break here rarely. If the * json format of a query being tested here then feel free to change this. */ -public class QueryBuilderBWCIT extends AbstractFullClusterRestartTestCase { - +public class QueryBuilderBWCIT extends ParameterizedFullClusterRestartTestCase { private static final List CANDIDATES = new ArrayList<>(); + protected static LocalClusterConfigProvider clusterConfig = c -> {}; + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(2) + .setting("xpack.security.enabled", "false") + .apply(() -> clusterConfig) + .build(); + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } + + public QueryBuilderBWCIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + static { addCandidate(""" "match": { "text_field": "value"} diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterHandle.java index 658925744860d..2a4e3e3958c57 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterHandle.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterHandle.java @@ -8,6 +8,8 @@ package org.elasticsearch.test.cluster; +import org.elasticsearch.test.cluster.util.Version; + import java.io.Closeable; /** @@ -73,4 +75,19 @@ public interface ClusterHandle extends Closeable { * @return cluster node TCP transport endpoints */ String getTransportEndpoint(int index); + + /** + * Upgrades a single node to the given version. Method blocks until the node is back up and ready to respond to requests. + * + * @param index index of node ot upgrade + * @param version version to upgrade to + */ + void upgradeNodeToVersion(int index, Version version); + + /** + * Performs a "full cluster restart" upgrade to the given version. Method blocks until the cluster is restarted and available. + * + * @param version version to upgrade to + */ + void upgradeToVersion(Version version); } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java index 2e3366cdf9af3..aa71ffdf27a72 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java @@ -12,6 +12,7 @@ import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.cluster.SettingsProvider; import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.Version; import org.elasticsearch.test.cluster.util.resource.Resource; import java.util.ArrayList; @@ -32,9 +33,11 @@ public abstract class AbstractLocalSpecBuilder> im private final Set plugins = new HashSet<>(); private final Set features = new HashSet<>(); private final Map keystoreSettings = new HashMap<>(); + private final Map keystoreFiles = new HashMap<>(); private final Map extraConfigFiles = new HashMap<>(); private final Map systemProperties = new HashMap<>(); private DistributionType distributionType; + private Version version; private String keystorePassword; protected AbstractLocalSpecBuilder(AbstractLocalSpecBuilder parent) { @@ -138,6 +141,16 @@ public Map getKeystoreSettings() { return inherit(() -> parent.getKeystoreSettings(), keystoreSettings); } + @Override + public T keystore(String key, Resource file) { + this.keystoreFiles.put(key, file); + return cast(this); + } + + public Map getKeystoreFiles() { + return inherit(() -> parent.getKeystoreFiles(), keystoreFiles); + } + @Override public T configFile(String fileName, Resource configFile) { this.extraConfigFiles.put(fileName, configFile); @@ -168,6 +181,16 @@ public String getKeystorePassword() { return inherit(() -> parent.getKeystorePassword(), keystorePassword); } + @Override + public T version(Version version) { + this.version = version; + return cast(this); + } + + public Version getVersion() { + return inherit(() -> parent.getVersion(), version); + } + private List inherit(Supplier> parent, List child) { List combinedList = new ArrayList<>(); if (this.parent != null) { diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterSpecBuilder.java index 9c4aa48eb03d4..fad762fa34c23 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterSpecBuilder.java @@ -19,12 +19,14 @@ import java.util.List; import java.util.Optional; import java.util.function.Consumer; +import java.util.function.Supplier; public class DefaultLocalClusterSpecBuilder extends AbstractLocalSpecBuilder implements LocalClusterSpecBuilder { private String name = "test-cluster"; private final List nodeBuilders = new ArrayList<>(); private final List users = new ArrayList<>(); private final List roleFiles = new ArrayList<>(); + private final List> lazyConfigProviders = new ArrayList<>(); public DefaultLocalClusterSpecBuilder() { super(null); @@ -46,6 +48,12 @@ public DefaultLocalClusterSpecBuilder apply(LocalClusterConfigProvider configPro return this; } + @Override + public LocalClusterSpecBuilder apply(Supplier configProvider) { + lazyConfigProviders.add(configProvider); + return this; + } + @Override public DefaultLocalClusterSpecBuilder nodes(int nodes) { if (nodes < nodeBuilders.size()) { @@ -117,7 +125,28 @@ public ElasticsearchCluster build() { clusterSpec.setNodes(nodeSpecs); clusterSpec.validate(); - return new LocalElasticsearchCluster(clusterSpec); + return new LocalElasticsearchCluster(this); + } + + LocalClusterSpec buildClusterSpec() { + // Apply lazily provided configuration + lazyConfigProviders.forEach(s -> s.get().apply(this)); + + List clusterUsers = users.isEmpty() ? List.of(User.DEFAULT_USER) : users; + LocalClusterSpec clusterSpec = new LocalClusterSpec(name, clusterUsers, roleFiles); + List nodeSpecs; + + if (nodeBuilders.isEmpty()) { + // No node-specific configuration so assume a single-node cluster + nodeSpecs = List.of(new DefaultLocalNodeSpecBuilder(this).build(clusterSpec)); + } else { + nodeSpecs = nodeBuilders.stream().map(node -> node.build(clusterSpec)).toList(); + } + + clusterSpec.setNodes(nodeSpecs); + clusterSpec.validate(); + + return clusterSpec; } public static class DefaultLocalNodeSpecBuilder extends AbstractLocalSpecBuilder implements LocalNodeSpecBuilder { @@ -138,7 +167,7 @@ private LocalNodeSpec build(LocalClusterSpec cluster) { return new LocalNodeSpec( cluster, name, - Version.CURRENT, + Optional.ofNullable(getVersion()).orElse(Version.CURRENT), getSettingsProviders(), getSettings(), getEnvironmentProviders(), @@ -148,6 +177,7 @@ private LocalNodeSpec build(LocalClusterSpec cluster) { Optional.ofNullable(getDistributionType()).orElse(DistributionType.INTEG_TEST), getFeatures(), getKeystoreSettings(), + getKeystoreFiles(), getKeystorePassword(), getExtraConfigFiles(), getSystemProperties() diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterFactory.java index 08318b5145038..5f43bb8aa71b6 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterFactory.java @@ -79,21 +79,20 @@ public LocalClusterHandle create(LocalClusterSpec spec) { public class Node { private final LocalNodeSpec spec; private final Path workingDir; - private final Path distributionDir; private final Path repoDir; private final Path dataDir; private final Path logsDir; private final Path configDir; private final Path tempDir; - private boolean initialized = false; + private Path distributionDir; + private Version currentVersion; private Process process = null; private DistributionDescriptor distributionDescriptor; public Node(LocalNodeSpec spec) { this.spec = spec; this.workingDir = baseWorkingDir.resolve(spec.getCluster().getName()).resolve(spec.getName()); - this.distributionDir = workingDir.resolve("distro"); // location of es distribution files, typically hard-linked this.repoDir = baseWorkingDir.resolve("repo"); this.dataDir = workingDir.resolve("data"); this.logsDir = workingDir.resolve("logs"); @@ -101,22 +100,30 @@ public Node(LocalNodeSpec spec) { this.tempDir = workingDir.resolve("tmp"); // elasticsearch temporary directory } - public synchronized void start() { + public synchronized void start(Version version) { LOGGER.info("Starting Elasticsearch node '{}'", spec.getName()); + if (version != null) { + spec.setVersion(version); + } - if (initialized == false) { + if (currentVersion == null || currentVersion.equals(spec.getVersion()) == false) { LOGGER.info("Creating installation for node '{}' in {}", spec.getName(), workingDir); distributionDescriptor = resolveDistribution(); LOGGER.info("Distribution for node '{}': {}", spec.getName(), distributionDescriptor); - initializeWorkingDirectory(); + distributionDir = OS.conditional( + // Use per-version distribution directories on Windows to avoid cleanup failures + c -> c.onWindows(() -> workingDir.resolve("distro").resolve(distributionDescriptor.getVersion().toString())) + .onUnix(() -> workingDir.resolve("distro")) + ); + initializeWorkingDirectory(currentVersion != null); createConfigDirectory(); copyExtraConfigFiles(); // extra config files might be needed for running cli tools like plugin install copyExtraJarFiles(); installPlugins(); - if (spec.getDistributionType() == DistributionType.INTEG_TEST) { + if (distributionDescriptor.getType() == DistributionType.INTEG_TEST) { installModules(); } - initialized = true; + currentVersion = spec.getVersion(); } else { createConfigDirectory(); copyExtraConfigFiles(); @@ -125,6 +132,7 @@ public synchronized void start() { writeConfiguration(); createKeystore(); addKeystoreSettings(); + addKeystoreFiles(); configureSecurity(); startElasticsearch(); @@ -135,6 +143,7 @@ public synchronized void stop(boolean forcibly) { ProcessUtils.stopHandle(process.toHandle(), forcibly); ProcessReaper.instance().unregister(getServiceName()); } + deletePortsFiles(); } public void waitForExit() { @@ -159,6 +168,20 @@ public String getTransportEndpoint() { return readPortsFile(portsFile).get(0); } + public void deletePortsFiles() { + try { + Path hostsFile = workingDir.resolve("config").resolve("unicast_hosts.txt"); + Path httpPortsFile = workingDir.resolve("logs").resolve("http.ports"); + Path transportPortsFile = workingDir.resolve("logs").resolve("transport.ports"); + + Files.deleteIfExists(hostsFile); + Files.deleteIfExists(httpPortsFile); + Files.deleteIfExists(transportPortsFile); + } catch (IOException e) { + throw new UncheckedIOException("Failed to write unicast_hosts for: " + this, e); + } + } + public LocalNodeSpec getSpec() { return spec; } @@ -205,9 +228,13 @@ private List readPortsFile(Path file) { } } - private void initializeWorkingDirectory() { + private void initializeWorkingDirectory(boolean preserveWorkingDirectory) { try { - IOUtils.deleteWithRetry(workingDir); + if (preserveWorkingDirectory) { + IOUtils.deleteWithRetry(distributionDir); + } else { + IOUtils.deleteWithRetry(workingDir); + } try { IOUtils.syncWithLinks(distributionDescriptor.getDistributionDir(), distributionDir); } catch (IOUtils.LinkCreationException e) { @@ -350,6 +377,31 @@ private void addKeystoreSettings() { }); } + private void addKeystoreFiles() { + spec.getKeystoreFiles().forEach((key, file) -> { + try { + Path path = Files.createTempFile(tempDir, key, null); + file.writeTo(path); + + ProcessUtils.exec( + spec.getKeystorePassword(), + workingDir, + OS.conditional( + c -> c.onWindows(() -> distributionDir.resolve("bin").resolve("elasticsearch-keystore.bat")) + .onUnix(() -> distributionDir.resolve("bin").resolve("elasticsearch-keystore")) + ), + getEnvironmentVariables(), + false, + "add-file", + key, + path.toString() + ).waitFor(); + } catch (InterruptedException | IOException e) { + throw new RuntimeException(e); + } + }); + } + private void configureSecurity() { if (spec.isSecurityEnabled()) { if (spec.getUsers().isEmpty() == false) { diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java index 878b017e3cd62..6ad2709957299 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java @@ -14,7 +14,7 @@ import org.elasticsearch.test.cluster.local.LocalClusterFactory.Node; import org.elasticsearch.test.cluster.local.model.User; import org.elasticsearch.test.cluster.util.ExceptionUtils; -import org.elasticsearch.test.cluster.util.Retry; +import org.elasticsearch.test.cluster.util.Version; import java.io.IOException; import java.io.UncheckedIOException; @@ -28,7 +28,6 @@ import java.util.concurrent.ForkJoinPool; import java.util.concurrent.ForkJoinWorkerThread; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -66,7 +65,7 @@ public LocalClusterHandle(String name, List nodes) { public void start() { if (started.getAndSet(true) == false) { LOGGER.info("Starting Elasticsearch test cluster '{}'", name); - execute(() -> nodes.parallelStream().forEach(Node::start)); + execute(() -> nodes.parallelStream().forEach(n -> n.start(null))); } waitUntilReady(); } @@ -75,11 +74,10 @@ public void start() { public void stop(boolean forcibly) { if (started.getAndSet(false)) { LOGGER.info("Stopping Elasticsearch test cluster '{}', forcibly: {}", name, forcibly); - execute(() -> nodes.forEach(n -> n.stop(forcibly))); - deletePortFiles(); + execute(() -> nodes.parallelStream().forEach(n -> n.stop(forcibly))); } else { // Make sure the process is stopped, otherwise wait - execute(() -> nodes.forEach(n -> n.waitForExit())); + execute(() -> nodes.parallelStream().forEach(Node::waitForExit)); } } @@ -128,16 +126,31 @@ public String getTransportEndpoint(int index) { return getTransportEndpoints().split(",")[index]; } + @Override + public void upgradeNodeToVersion(int index, Version version) { + Node node = nodes.get(index); + node.stop(false); + LOGGER.info("Upgrading node '{}' to version {}", node.getSpec().getName(), version); + node.start(version); + waitUntilReady(); + } + + @Override + public void upgradeToVersion(Version version) { + stop(false); + if (started.getAndSet(true) == false) { + LOGGER.info("Upgrading Elasticsearch test cluster '{}' to version {}", name, version); + execute(() -> nodes.parallelStream().forEach(n -> n.start(version))); + } + waitUntilReady(); + } + private void waitUntilReady() { writeUnicastHostsFile(); try { - Retry.retryUntilTrue(CLUSTER_UP_TIMEOUT, Duration.ZERO, () -> { - WaitForHttpResource wait = configureWaitForReady(); - return wait.wait(500); - }); - } catch (TimeoutException e) { - throw new RuntimeException("Timed out after " + CLUSTER_UP_TIMEOUT + " waiting for cluster '" + name + "' status to be yellow"); - } catch (ExecutionException e) { + WaitForHttpResource wait = configureWaitForReady(); + wait.waitFor(CLUSTER_UP_TIMEOUT.toMillis()); + } catch (Exception e) { throw new RuntimeException("An error occurred while checking cluster '" + name + "' status.", e); } } @@ -191,7 +204,7 @@ private boolean isSecurityAutoConfigured(Node node) { private void writeUnicastHostsFile() { String transportUris = execute(() -> nodes.parallelStream().map(Node::getTransportEndpoint).collect(Collectors.joining("\n"))); - nodes.forEach(node -> { + execute(() -> nodes.parallelStream().forEach(node -> { try { Path hostsFile = node.getWorkingDir().resolve("config").resolve("unicast_hosts.txt"); if (Files.notExists(hostsFile)) { @@ -200,23 +213,7 @@ private void writeUnicastHostsFile() { } catch (IOException e) { throw new UncheckedIOException("Failed to write unicast_hosts for: " + node, e); } - }); - } - - private void deletePortFiles() { - nodes.forEach(node -> { - try { - Path hostsFile = node.getWorkingDir().resolve("config").resolve("unicast_hosts.txt"); - Path httpPortsFile = node.getWorkingDir().resolve("logs").resolve("http.ports"); - Path tranportPortsFile = node.getWorkingDir().resolve("logs").resolve("transport.ports"); - - Files.deleteIfExists(hostsFile); - Files.deleteIfExists(httpPortsFile); - Files.deleteIfExists(tranportPortsFile); - } catch (IOException e) { - throw new UncheckedIOException("Failed to write unicast_hosts for: " + node, e); - } - }); + })); } private T execute(Callable task) { diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java index 2836411bbb067..2234b037381a8 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java @@ -69,7 +69,6 @@ void validate() { public static class LocalNodeSpec { private final LocalClusterSpec cluster; private final String name; - private final Version version; private final List settingsProviders; private final Map settings; private final List environmentProviders; @@ -79,9 +78,11 @@ public static class LocalNodeSpec { private final DistributionType distributionType; private final Set features; private final Map keystoreSettings; + private final Map keystoreFiles; private final String keystorePassword; private final Map extraConfigFiles; private final Map systemProperties; + private Version version; public LocalNodeSpec( LocalClusterSpec cluster, @@ -96,6 +97,7 @@ public LocalNodeSpec( DistributionType distributionType, Set features, Map keystoreSettings, + Map keystoreFiles, String keystorePassword, Map extraConfigFiles, Map systemProperties @@ -112,11 +114,16 @@ public LocalNodeSpec( this.distributionType = distributionType; this.features = features; this.keystoreSettings = keystoreSettings; + this.keystoreFiles = keystoreFiles; this.keystorePassword = keystorePassword; this.extraConfigFiles = extraConfigFiles; this.systemProperties = systemProperties; } + void setVersion(Version version) { + this.version = version; + } + public LocalClusterSpec getCluster() { return cluster; } @@ -157,6 +164,10 @@ public Map getKeystoreSettings() { return keystoreSettings; } + public Map getKeystoreFiles() { + return keystoreFiles; + } + public String getKeystorePassword() { return keystorePassword; } @@ -254,6 +265,7 @@ private LocalNodeSpec getFilteredSpec(SettingsProvider filteredProvider) { n.distributionType, n.features, n.keystoreSettings, + n.keystoreFiles, n.keystorePassword, n.extraConfigFiles, n.systemProperties diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpecBuilder.java index c07a491d2ace6..1f4086fd47fe8 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpecBuilder.java @@ -12,6 +12,7 @@ import org.elasticsearch.test.cluster.util.resource.Resource; import java.util.function.Consumer; +import java.util.function.Supplier; public interface LocalClusterSpecBuilder extends LocalSpecBuilder { /** @@ -19,8 +20,18 @@ public interface LocalClusterSpecBuilder extends LocalSpecBuilder configProvider); + /** * Sets the number of nodes for the cluster. */ diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalElasticsearchCluster.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalElasticsearchCluster.java index 54d541cd07144..9a5e5666f5e9a 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalElasticsearchCluster.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalElasticsearchCluster.java @@ -10,18 +10,21 @@ import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.LocalDistributionResolver; +import org.elasticsearch.test.cluster.local.distribution.ReleasedDistributionResolver; import org.elasticsearch.test.cluster.local.distribution.SnapshotDistributionResolver; +import org.elasticsearch.test.cluster.util.Version; import org.junit.runner.Description; import org.junit.runners.model.Statement; import java.nio.file.Path; public class LocalElasticsearchCluster implements ElasticsearchCluster { - private final LocalClusterSpec spec; + private final DefaultLocalClusterSpecBuilder builder; + private LocalClusterSpec spec; private LocalClusterHandle handle; - public LocalElasticsearchCluster(LocalClusterSpec spec) { - this.spec = spec; + public LocalElasticsearchCluster(DefaultLocalClusterSpecBuilder builder) { + this.builder = builder; } @Override @@ -30,9 +33,10 @@ public Statement apply(Statement base, Description description) { @Override public void evaluate() throws Throwable { try { + spec = builder.buildClusterSpec(); handle = new LocalClusterFactory( Path.of(System.getProperty("java.io.tmpdir")).resolve(description.getDisplayName()).toAbsolutePath(), - new LocalDistributionResolver(new SnapshotDistributionResolver()) + new LocalDistributionResolver(new SnapshotDistributionResolver(new ReleasedDistributionResolver())) ).create(spec); handle.start(); base.evaluate(); @@ -97,6 +101,18 @@ public String getTransportEndpoint(int index) { return handle.getTransportEndpoint(index); } + @Override + public void upgradeNodeToVersion(int index, Version version) { + checkHandle(); + handle.upgradeNodeToVersion(index, version); + } + + @Override + public void upgradeToVersion(Version version) { + checkHandle(); + handle.upgradeToVersion(version); + } + private void checkHandle() { if (handle == null) { throw new IllegalStateException("Cluster handle has not been initialized. Did you forget the @ClassRule annotation?"); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java index 3b9428bc1a1aa..d01d7d303748f 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java @@ -12,6 +12,7 @@ import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.cluster.SettingsProvider; import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.Version; import org.elasticsearch.test.cluster.util.resource.Resource; import java.util.function.Supplier; @@ -68,6 +69,11 @@ interface LocalSpecBuilder> { */ T keystore(String key, String value); + /** + * Adds a secure file to the node keystore. + */ + T keystore(String key, Resource file); + /** * Sets the security setting keystore password. */ @@ -78,6 +84,11 @@ interface LocalSpecBuilder> { */ T configFile(String fileName, Resource configFile); + /** + * Sets the version of Elasticsearch. Defaults to {@link Version#CURRENT}. + */ + T version(Version version); + /** * Adds a system property to node JVM arguments. */ diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/WaitForHttpResource.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/WaitForHttpResource.java index edab2cdf1e7e9..f00e6f13cb314 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/WaitForHttpResource.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/WaitForHttpResource.java @@ -90,7 +90,7 @@ public void setPassword(String password) { this.password = password; } - public boolean wait(int durationInMs) throws GeneralSecurityException, InterruptedException, IOException { + public boolean waitFor(long durationInMs) throws GeneralSecurityException, InterruptedException, IOException { final long waitUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durationInMs); final long sleep = Long.max(durationInMs / 10, 100); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/LocalDistributionResolver.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/LocalDistributionResolver.java index 5c9f45cbe092f..b9442b28e1591 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/LocalDistributionResolver.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/LocalDistributionResolver.java @@ -13,6 +13,9 @@ import java.nio.file.Files; import java.nio.file.Path; +/** + * A {@link DistributionResolver} for resolving locally built distributions for the current version of Elasticsearch. + */ public class LocalDistributionResolver implements DistributionResolver { private final DistributionResolver delegate; diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/ReleasedDistributionResolver.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/ReleasedDistributionResolver.java new file mode 100644 index 0000000000000..12654be310ef8 --- /dev/null +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/ReleasedDistributionResolver.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test.cluster.local.distribution; + +import org.elasticsearch.test.cluster.util.Version; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * A {@link DistributionResolver} for resolving previously released distributions of Elasticsearch. + */ +public class ReleasedDistributionResolver implements DistributionResolver { + private static final String BWC_DISTRIBUTION_SYSPROP_PREFIX = "tests.release.distribution."; + + @Override + public DistributionDescriptor resolve(Version version, DistributionType type) { + String distributionPath = System.getProperty(BWC_DISTRIBUTION_SYSPROP_PREFIX + version.toString()); + + if (distributionPath == null) { + String taskPath = System.getProperty("tests.task"); + String project = taskPath.substring(0, taskPath.lastIndexOf(':')); + String taskName = taskPath.substring(taskPath.lastIndexOf(':') + 1); + + throw new IllegalStateException( + "Cannot locate Elasticsearch distribution. Ensure you've added the following to the build script for project '" + + project + + "':\n\n" + + "tasks.named('" + + taskName + + "') {\n" + + " usesBwcDistribution(" + + version + + ")\n" + + "}" + ); + } + + Path distributionDir = Path.of(distributionPath); + if (Files.notExists(distributionDir)) { + throw new IllegalStateException( + "Cannot locate Elasticsearch distribution. Directory at '" + distributionDir + "' does not exist." + ); + } + + return new DefaultDistributionDescriptor(version, false, distributionDir, DistributionType.DEFAULT); + } +} diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/SnapshotDistributionResolver.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/SnapshotDistributionResolver.java index 182dbe66a584d..c6cecf09e9b9d 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/SnapshotDistributionResolver.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/SnapshotDistributionResolver.java @@ -10,10 +10,36 @@ import org.elasticsearch.test.cluster.util.Version; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * A {@link DistributionResolver} for resolving snapshot versions of Elasticsearch for previous, backwards-compatible versions. + */ public class SnapshotDistributionResolver implements DistributionResolver { + private static final String BWC_DISTRIBUTION_SYSPROP_PREFIX = "tests.snapshot.distribution."; + private final DistributionResolver delegate; + + public SnapshotDistributionResolver(DistributionResolver delegate) { + this.delegate = delegate; + } + @Override public DistributionDescriptor resolve(Version version, DistributionType type) { - // Not yet implemented - throw new UnsupportedOperationException("Cannot resolve distribution for version " + version); + String distributionPath = System.getProperty(BWC_DISTRIBUTION_SYSPROP_PREFIX + version.toString()); + + if (distributionPath != null) { + Path distributionDir = Path.of(distributionPath); + if (Files.notExists(distributionDir)) { + throw new IllegalStateException( + "Cannot locate Elasticsearch distribution. Directory at '" + distributionDir + "' does not exist." + ); + } + + // Snapshot distributions are never release builds and always use the default distribution + return new DefaultDistributionDescriptor(version, true, distributionDir, DistributionType.DEFAULT); + } + + return delegate.resolve(version, type); } } diff --git a/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle b/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle index 429b29bbc9fdb..d9539bb668b4b 100644 --- a/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle +++ b/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle @@ -1,104 +1,21 @@ -import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask -apply plugin: 'elasticsearch.internal-testclusters' -apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.bwc-test' dependencies { // TODO: Remove core dependency and change tests to not use builders that are part of xpack-core. // Currently needed for MlConfigIndexMappingsFullClusterRestartIT and SLM classes used in // FullClusterRestartIT - testImplementation(testArtifact(project(xpackModule('core')))) - testImplementation(testArtifact(project(":qa:full-cluster-restart"))) - testImplementation project(':x-pack:qa') -} - -tasks.named("forbiddenPatterns") { - exclude '**/system_key' -} - -String outputDir = "${buildDir}/generated-resources/${project.name}" - -tasks.register("copyTestNodeKeyMaterial", Copy) { - from project(':x-pack:plugin:core') - .files( - 'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem', - 'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt' - ) - into outputDir + javaRestTestImplementation(testArtifact(project(xpackModule('core')))) + javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart"))) + javaRestTestImplementation project(':x-pack:qa') } BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> - def baseCluster = testClusters.register(baseName) { - testDistribution = "DEFAULT" - if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) { - // When testing older versions we have to first upgrade to 7.last - versions = [bwcVersion.toString(), BuildParams.bwcVersions.minimumWireCompatibleVersion.toString(), project.version] - } else { - versions = [bwcVersion.toString(), project.version] - } - numberOfNodes = 2 - setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" - user username: "test_user", password: "x-pack-test-password" - - setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" - // some tests rely on the translog not being flushed - setting 'indices.memory.shard_inactive_time', '60m' - setting 'xpack.security.enabled', 'true' - setting 'xpack.security.transport.ssl.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - - extraConfigFile 'testnode.pem', file("${outputDir}/testnode.pem") - extraConfigFile 'testnode.crt', file("${outputDir}/testnode.crt") - - keystore 'xpack.watcher.encryption_key', file("${project.projectDir}/src/test/resources/system_key") - setting 'xpack.watcher.encrypt_sensitive_data', 'true' - - setting 'xpack.security.transport.ssl.key', 'testnode.pem' - setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' - keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' - - setting 'xpack.security.authc.api_key.enabled', 'true' - - requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") - } - - tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { - mustRunAfter("precommit") - useCluster baseCluster - dependsOn "copyTestNodeKeyMaterial" - doFirst { - delete("${buildDir}/cluster/shared/repo/${baseName}") - } - systemProperty 'tests.is_old_cluster', 'true' - } - - tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) { - mustRunAfter("precommit") - useCluster baseCluster - dependsOn "${baseName}#oldClusterTest" - doFirst { - baseCluster.get().goToNextVersion() - if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) { - // When doing a full cluster restart of older versions we actually have to upgrade twice. First to 7.last, then to the current version. - baseCluster.get().goToNextVersion() - } - } - systemProperty 'tests.is_old_cluster', 'false' - } - - String oldVersion = bwcVersion.toString().minus("-SNAPSHOT") - tasks.matching { it.name.startsWith("${baseName}#") && it.name.endsWith("ClusterTest") }.configureEach { - it.systemProperty 'tests.old_cluster_version', oldVersion - it.systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}" - it.nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) - it.nonInputProperties.systemProperty('tests.clustername', baseName) - } - - tasks.register(bwcTaskName(bwcVersion)) { - dependsOn "${baseName}#upgradedClusterTest" - } - + tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { + usesBwcDistribution(bwcVersion) + systemProperty("tests.old_cluster_version", bwcVersion) + } } diff --git a/x-pack/plugin/shutdown/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java similarity index 65% rename from x-pack/plugin/shutdown/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java rename to x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index df6e3ed6b9388..07ed594770649 100644 --- a/x-pack/plugin/shutdown/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.restart; +import com.carrotsearch.randomizedtesting.annotations.Name; + import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -13,10 +15,17 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.resource.Resource; import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; +import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.BeforeClass; +import org.junit.ClassRule; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -33,7 +42,37 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.notNullValue; -public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase { +public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(2) + // some tests rely on the translog not being flushed + .setting("indices.memory.shard_inactive_time", "60m") + .setting("xpack.security.enabled", "true") + .setting("xpack.security.transport.ssl.enabled", "true") + .setting("xpack.security.transport.ssl.key", "testnode.pem") + .setting("xpack.security.transport.ssl.certificate", "testnode.crt") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.watcher.encrypt_sensitive_data", "true") + .setting("xpack.security.authc.api_key.enabled", "true") + .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem")) + .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt")) + .keystore("xpack.watcher.encryption_key", Resource.fromClasspath("system_key")) + .keystore("xpack.security.transport.ssl.secure_key_passphrase", "testnode") + .feature(FeatureFlag.TIME_SERIES_MODE) + .build(); + + public FullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } @Override protected Settings restClientSettings() { @@ -47,10 +86,13 @@ protected Settings restClientSettings() { .build(); } - @SuppressWarnings("unchecked") - public void testNodeShutdown() throws Exception { + @BeforeClass + public static void checkClusterVersion() { assumeTrue("no shutdown in versions before " + Version.V_7_15_0, getOldClusterVersion().onOrAfter(Version.V_7_15_0)); + } + @SuppressWarnings("unchecked") + public void testNodeShutdown() throws Exception { if (isRunningAgainstOldCluster()) { final Request getNodesReq = new Request("GET", "_nodes"); final Response getNodesResp = adminClient().performRequest(getNodesReq); @@ -64,7 +106,7 @@ public void testNodeShutdown() throws Exception { // Use the types available from as early as possible final String type = randomFrom("restart", "remove"); putBody.field("type", type); - putBody.field("reason", this.getTestName()); + putBody.field("reason", getRootTestName()); } putBody.endObject(); putShutdownRequest.setJsonEntity(Strings.toString(putBody)); @@ -81,7 +123,7 @@ public void testNodeShutdown() throws Exception { assertThat("there should be exactly one shutdown registered", shutdowns, hasSize(1)); final Map shutdown = shutdowns.get(0); assertThat(shutdown.get("node_id"), notNullValue()); // Since we randomly determine the node ID, we can't check it - assertThat(shutdown.get("reason"), equalTo(this.getTestName())); + assertThat(shutdown.get("reason"), equalTo(getRootTestName())); assertThat( (String) shutdown.get("status"), anyOf( diff --git a/x-pack/plugin/shutdown/qa/full-cluster-restart/src/test/resources/system_key b/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/resources/system_key similarity index 100% rename from x-pack/plugin/shutdown/qa/full-cluster-restart/src/test/resources/system_key rename to x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/resources/system_key diff --git a/x-pack/qa/full-cluster-restart/build.gradle b/x-pack/qa/full-cluster-restart/build.gradle index 3923d439d394d..d9539bb668b4b 100644 --- a/x-pack/qa/full-cluster-restart/build.gradle +++ b/x-pack/qa/full-cluster-restart/build.gradle @@ -1,110 +1,21 @@ -import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask -apply plugin: 'elasticsearch.internal-testclusters' -apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.bwc-test' dependencies { // TODO: Remove core dependency and change tests to not use builders that are part of xpack-core. // Currently needed for MlConfigIndexMappingsFullClusterRestartIT and SLM classes used in // FullClusterRestartIT - testImplementation(testArtifact(project(xpackModule('core')))) - testImplementation(testArtifact(project(":qa:full-cluster-restart"))) - testImplementation project(':x-pack:qa') -} - -tasks.named("forbiddenPatterns") { - exclude '**/system_key' -} - -String outputDir = "${buildDir}/generated-resources/${project.name}" - -tasks.register("copyTestNodeKeyMaterial", Copy) { - from project(':x-pack:plugin:core') - .files( - 'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem', - 'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt' - ) - into outputDir + javaRestTestImplementation(testArtifact(project(xpackModule('core')))) + javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart"))) + javaRestTestImplementation project(':x-pack:qa') } BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> - def baseCluster = testClusters.register(baseName) { - testDistribution = "DEFAULT" - if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) { - // When testing older versions we have to first upgrade to 7.last - versions = [bwcVersion.toString(), BuildParams.bwcVersions.minimumWireCompatibleVersion.toString(), project.version] - } else { - versions = [bwcVersion.toString(), project.version] - } - numberOfNodes = 2 - setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" - user username: "test_user", password: "x-pack-test-password" - - setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" - // some tests rely on the translog not being flushed - setting 'indices.memory.shard_inactive_time', '60m' - setting 'xpack.security.enabled', 'true' - setting 'xpack.security.transport.ssl.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - - extraConfigFile 'testnode.pem', file("${outputDir}/testnode.pem") - extraConfigFile 'testnode.crt', file("${outputDir}/testnode.crt") - - keystore 'xpack.watcher.encryption_key', file("${project.projectDir}/src/test/resources/system_key") - setting 'xpack.watcher.encrypt_sensitive_data', 'true' - - setting 'xpack.security.transport.ssl.key', 'testnode.pem' - setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' - keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' - - setting 'xpack.security.authc.api_key.enabled', 'true' - - requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") - } - - tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { - mustRunAfter("precommit") - useCluster baseCluster - dependsOn "copyTestNodeKeyMaterial" - doFirst { - delete("${buildDir}/cluster/shared/repo/${baseName}") - } - systemProperty 'tests.is_old_cluster', 'true' - exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class' - exclude 'org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.class' - exclude 'org/elasticsearch/upgrades/QueryBuilderBWCIT.class' - } - - tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) { - mustRunAfter("precommit") - useCluster baseCluster - dependsOn "${baseName}#oldClusterTest" - doFirst { - baseCluster.get().goToNextVersion() - if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) { - // When doing a full cluster restart of older versions we actually have to upgrade twice. First to 7.last, then to the current version. - baseCluster.get().goToNextVersion() - } - } - systemProperty 'tests.is_old_cluster', 'false' - exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class' - exclude 'org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.class' - exclude 'org/elasticsearch/upgrades/QueryBuilderBWCIT.class' - } - - String oldVersion = bwcVersion.toString().minus("-SNAPSHOT") - tasks.matching { it.name.startsWith("${baseName}#") && it.name.endsWith("ClusterTest") }.configureEach { - it.systemProperty 'tests.old_cluster_version', oldVersion - it.systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}" - it.nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) - it.nonInputProperties.systemProperty('tests.clustername', baseName) - } - - tasks.register(bwcTaskName(bwcVersion)) { - dependsOn "${baseName}#upgradedClusterTest" - } - + tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { + usesBwcDistribution(bwcVersion) + systemProperty("tests.old_cluster_version", bwcVersion) + } } diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/AbstractXpackFullClusterRestartTestCase.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/AbstractXpackFullClusterRestartTestCase.java new file mode 100644 index 0000000000000..0bc9101301a54 --- /dev/null +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/AbstractXpackFullClusterRestartTestCase.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.restart; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; +import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase; +import org.junit.ClassRule; + +public abstract class AbstractXpackFullClusterRestartTestCase extends ParameterizedFullClusterRestartTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(2) + // some tests rely on the translog not being flushed + .setting("indices.memory.shard_inactive_time", "60m") + .setting("xpack.security.enabled", "true") + .setting("xpack.security.transport.ssl.enabled", "true") + .setting("xpack.security.transport.ssl.key", "testnode.pem") + .setting("xpack.security.transport.ssl.certificate", "testnode.crt") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.watcher.encrypt_sensitive_data", "true") + .setting("xpack.security.authc.api_key.enabled", "true") + .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem")) + .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt")) + .keystore("xpack.watcher.encryption_key", Resource.fromClasspath("system_key")) + .keystore("xpack.security.transport.ssl.secure_key_passphrase", "testnode") + .feature(FeatureFlag.TIME_SERIES_MODE) + .build(); + + public AbstractXpackFullClusterRestartTestCase(FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } +} diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/CoreFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/CoreFullClusterRestartIT.java new file mode 100644 index 0000000000000..dcdc127079637 --- /dev/null +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/CoreFullClusterRestartIT.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.restart; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.upgrades.FullClusterRestartIT; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class CoreFullClusterRestartIT extends FullClusterRestartIT { + + static { + clusterConfig = c -> c.setting("xpack.security.enabled", "true") + .setting("xpack.security.transport.ssl.enabled", "true") + .setting("xpack.security.transport.ssl.key", "testnode.pem") + .setting("xpack.security.transport.ssl.certificate", "testnode.crt") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.watcher.encrypt_sensitive_data", "true") + .setting("xpack.security.authc.api_key.enabled", "true") + .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem")) + .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt")) + .keystore("xpack.watcher.encryption_key", Resource.fromClasspath("system_key")) + .keystore("xpack.security.transport.ssl.secure_key_passphrase", "testnode"); + } + + public CoreFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + + @Override + protected Settings restClientSettings() { + String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + +} diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java similarity index 96% rename from x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java rename to x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 42c551b16655b..ab48825ed983a 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.restart; +import com.carrotsearch.randomizedtesting.annotations.Name; + import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; @@ -25,7 +27,7 @@ import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.test.StreamsUtils; import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; import org.elasticsearch.xcontent.ObjectPath; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -61,11 +63,15 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.startsWith; -public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase { +public class FullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase { public static final int UPGRADE_FIELD_EXPECTED_INDEX_FORMAT_VERSION = 6; public static final int SECURITY_EXPECTED_INDEX_FORMAT_VERSION = 6; + public FullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + @Override protected Settings restClientSettings() { String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); @@ -103,12 +109,7 @@ public void testSecurityNativeRealm() throws Exception { } else { waitForYellow(".security"); final Request getSettingsRequest = new Request("GET", "/.security/_settings/index.format"); - getSettingsRequest.setOptions( - expectWarnings( - "this request accesses system indices: [.security-7], but in a future major " - + "version, direct access to system indices will be prevented by default" - ) - ); + getSettingsRequest.setOptions(systemIndexWarningHandlerOptions(".security-7")); Response settingsResponse = client().performRequest(getSettingsRequest); Map settingsResponseMap = entityAsMap(settingsResponse); logger.info("settings response map {}", settingsResponseMap); @@ -390,12 +391,7 @@ public void testApiKeySuperuser() throws IOException { "doc_type": "foo" }"""); if (getOldClusterVersion().onOrAfter(Version.V_7_10_0)) { - indexRequest.setOptions( - expectWarnings( - "this request accesses system indices: [.security-7], but in a future major " - + "version, direct access to system indices will be prevented by default" - ).toBuilder().addHeader("Authorization", apiKeyAuthHeader) - ); + indexRequest.setOptions(systemIndexWarningHandlerOptions(".security-7").addHeader("Authorization", apiKeyAuthHeader)); } else { indexRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", apiKeyAuthHeader)); } @@ -409,12 +405,7 @@ public void testApiKeySuperuser() throws IOException { // read is ok final Request searchRequest = new Request("GET", ".security/_search"); - searchRequest.setOptions( - expectWarnings( - "this request accesses system indices: [.security-7], but in a future major " - + "version, direct access to system indices will be prevented by default" - ).toBuilder().addHeader("Authorization", apiKeyAuthHeader) - ); + searchRequest.setOptions(systemIndexWarningHandlerOptions(".security-7").addHeader("Authorization", apiKeyAuthHeader)); assertOK(client().performRequest(searchRequest)); // write must not be allowed @@ -423,12 +414,7 @@ public void testApiKeySuperuser() throws IOException { { "doc_type": "foo" }"""); - indexRequest.setOptions( - expectWarnings( - "this request accesses system indices: [.security-7], but in a future major " - + "version, direct access to system indices will be prevented by default" - ).toBuilder().addHeader("Authorization", apiKeyAuthHeader) - ); + indexRequest.setOptions(systemIndexWarningHandlerOptions(".security-7").addHeader("Authorization", apiKeyAuthHeader)); final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(indexRequest)); assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(403)); assertThat(e.getMessage(), containsString("is unauthorized")); @@ -995,4 +981,17 @@ private static void createComposableTemplate(RestClient client, String templateN createIndexTemplateRequest.setEntity(templateJSON); client.performRequest(createIndexTemplateRequest); } + + private RequestOptions.Builder systemIndexWarningHandlerOptions(String index) { + return RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler( + w -> w.size() > 0 + && w.contains( + "this request accesses system indices: [" + + index + + "], but in a future major " + + "version, direct access to system indices will be prevented by default" + ) == false + ); + } } diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java similarity index 97% rename from x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java rename to x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java index 4d1a5dfb75ab7..25a14c47e52c7 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.restart; +import com.carrotsearch.randomizedtesting.annotations.Name; + import org.apache.http.util.EntityUtils; import org.elasticsearch.Version; import org.elasticsearch.client.Request; @@ -15,7 +17,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Strings; -import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; import org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus; import org.junit.Before; @@ -32,7 +34,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; -public class MLModelDeploymentFullClusterRestartIT extends AbstractFullClusterRestartTestCase { +public class MLModelDeploymentFullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase { // See PyTorchModelIT for how this model was created static final String BASE_64_ENCODED_MODEL = @@ -63,6 +65,10 @@ public class MLModelDeploymentFullClusterRestartIT extends AbstractFullClusterRe RAW_MODEL_SIZE = Base64.getDecoder().decode(BASE_64_ENCODED_MODEL).length; } + public MLModelDeploymentFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + @Before public void setLogging() throws IOException { Request loggingSettings = new Request("PUT", "_cluster/settings"); diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java similarity index 94% rename from x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java rename to x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java index bfc078ffe9206..e4ab3957f2627 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.restart; +import com.carrotsearch.randomizedtesting.annotations.Name; + import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -13,7 +15,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Strings; -import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; import org.elasticsearch.xpack.test.rest.IndexMappingTemplateAsserter; import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; @@ -29,11 +31,15 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClusterRestartTestCase { +public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase { private static final String OLD_CLUSTER_JOB_ID = "ml-config-mappings-old-cluster-job"; private static final String NEW_CLUSTER_JOB_ID = "ml-config-mappings-new-cluster-job"; + public MlConfigIndexMappingsFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + @Override protected Settings restClientSettings() { String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java similarity index 96% rename from x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java rename to x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 274fa7ee40fce..aeb3dad547946 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.restart; +import com.carrotsearch.randomizedtesting.annotations.Name; + import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; @@ -16,7 +18,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Strings; import org.elasticsearch.core.Tuple; -import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.spi.XContentProvider; @@ -38,7 +40,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class MlHiddenIndicesFullClusterRestartIT extends AbstractFullClusterRestartTestCase { +public class MlHiddenIndicesFullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase { private static final String JOB_ID = "ml-hidden-indices-old-cluster-job"; private static final List, String>> EXPECTED_INDEX_ALIAS_PAIRS = List.of( @@ -49,6 +51,10 @@ public class MlHiddenIndicesFullClusterRestartIT extends AbstractFullClusterRest Tuple.tuple(List.of(".ml-anomalies-shared"), ".ml-anomalies-.write-" + JOB_ID) ); + public MlHiddenIndicesFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + @Override protected Settings restClientSettings() { String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java similarity index 96% rename from x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java rename to x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java index 61ce6f7827e2a..2bbda9123ae34 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.restart; +import com.carrotsearch.randomizedtesting.annotations.Name; + import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -17,7 +19,7 @@ import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; -import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; @@ -35,13 +37,17 @@ import static org.hamcrest.Matchers.emptyOrNullString; import static org.hamcrest.Matchers.is; -public class MlMigrationFullClusterRestartIT extends AbstractFullClusterRestartTestCase { +public class MlMigrationFullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase { private static final String OLD_CLUSTER_OPEN_JOB_ID = "migration-old-cluster-open-job"; private static final String OLD_CLUSTER_STARTED_DATAFEED_ID = "migration-old-cluster-started-datafeed"; private static final String OLD_CLUSTER_CLOSED_JOB_ID = "migration-old-cluster-closed-job"; private static final String OLD_CLUSTER_STOPPED_DATAFEED_ID = "migration-old-cluster-stopped-datafeed"; + public MlMigrationFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + @Override protected Settings restClientSettings() { String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/QueryBuilderBWCIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/QueryBuilderBWCIT.java new file mode 100644 index 0000000000000..563cde322b725 --- /dev/null +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/QueryBuilderBWCIT.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.restart; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class QueryBuilderBWCIT extends org.elasticsearch.upgrades.QueryBuilderBWCIT { + + static { + clusterConfig = c -> c.setting("xpack.security.enabled", "true") + .setting("xpack.security.transport.ssl.enabled", "true") + .setting("xpack.security.transport.ssl.key", "testnode.pem") + .setting("xpack.security.transport.ssl.certificate", "testnode.crt") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.watcher.encrypt_sensitive_data", "true") + .setting("xpack.security.authc.api_key.enabled", "true") + .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem")) + .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt")) + .keystore("xpack.watcher.encryption_key", Resource.fromClasspath("system_key")) + .keystore("xpack.security.transport.ssl.secure_key_passphrase", "testnode"); + } + + public QueryBuilderBWCIT(FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + + @Override + protected Settings restClientSettings() { + String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } +} diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java similarity index 89% rename from x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java rename to x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java index ea926e964360d..043b3f49a8825 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.restart; +import com.carrotsearch.randomizedtesting.annotations.Name; + import org.apache.http.util.EntityUtils; import org.elasticsearch.Version; import org.elasticsearch.client.Request; @@ -14,7 +16,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; import java.nio.charset.StandardCharsets; import java.util.Base64; @@ -23,7 +25,11 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.not; -public class WatcherMappingUpdateIT extends AbstractFullClusterRestartTestCase { +public class WatcherMappingUpdateIT extends AbstractXpackFullClusterRestartTestCase { + + public WatcherMappingUpdateIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } @Override protected Settings restClientSettings() { @@ -91,7 +97,7 @@ private void assertNoMappingVersion(String index) throws Exception { private RequestOptions.Builder getWarningHandlerOptions(String index) { return RequestOptions.DEFAULT.toBuilder() - .setWarningsHandler(w -> w.contains(getWatcherSystemIndexWarning(index)) == false || w.size() != 1); + .setWarningsHandler(w -> w.size() > 0 && w.contains(getWatcherSystemIndexWarning(index)) == false); } private String getWatcherSystemIndexWarning(String index) { diff --git a/x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/funny-timeout-watch.json b/x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/funny-timeout-watch.json similarity index 100% rename from x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/funny-timeout-watch.json rename to x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/funny-timeout-watch.json diff --git a/x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/logging-watch.json b/x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/logging-watch.json similarity index 100% rename from x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/logging-watch.json rename to x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/logging-watch.json diff --git a/x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/simple-watch.json b/x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/simple-watch.json similarity index 100% rename from x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/simple-watch.json rename to x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/simple-watch.json diff --git a/x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/throttle-period-watch.json b/x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/throttle-period-watch.json similarity index 100% rename from x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/throttle-period-watch.json rename to x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/throttle-period-watch.json diff --git a/x-pack/qa/full-cluster-restart/src/test/resources/system_key b/x-pack/qa/full-cluster-restart/src/javaRestTest/resources/system_key similarity index 100% rename from x-pack/qa/full-cluster-restart/src/test/resources/system_key rename to x-pack/qa/full-cluster-restart/src/javaRestTest/resources/system_key diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/CoreFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/CoreFullClusterRestartIT.java deleted file mode 100644 index e06cb12f747a7..0000000000000 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/CoreFullClusterRestartIT.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.restart; - -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.upgrades.FullClusterRestartIT; - -import java.nio.charset.StandardCharsets; -import java.util.Base64; - -public class CoreFullClusterRestartIT extends FullClusterRestartIT { - - @Override - protected Settings restClientSettings() { - String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - -} diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/QueryBuilderBWCIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/QueryBuilderBWCIT.java deleted file mode 100644 index cffc6881df645..0000000000000 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/QueryBuilderBWCIT.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.restart; - -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; - -import java.nio.charset.StandardCharsets; -import java.util.Base64; - -public class QueryBuilderBWCIT extends org.elasticsearch.upgrades.QueryBuilderBWCIT { - - @Override - protected Settings restClientSettings() { - String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } -}