Convert full cluster restart tests to new rest testing framework

build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle

@@ -9,6 +9,8 @@
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin
 import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 ext.bwcTaskName = { Version version ->
   return "v${version}#bwcTest"
@@ -36,5 +38,17 @@ plugins.withType(ElasticsearchTestBasePlugin) {
   }
 }
 
+plugins.withType(InternalJavaRestTestPlugin) {
+  tasks.named("javaRestTest") {
+    enabled = false
+  }
+
+  tasks.withType(StandaloneRestIntegTestTask).configureEach {
+    testClassesDirs = sourceSets.javaRestTest.output.classesDirs
+    classpath = sourceSets.javaRestTest.runtimeClasspath
+    usesDefaultDistribution()
+  }
+}
+
 tasks.matching { it.name.equals("check") }.configureEach {dependsOn(bwcTestSnapshots) }
 tasks.matching { it.name.equals("test") }.configureEach {enabled = false}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -89,7 +89,7 @@ public void execute(Task t) {
             test.getJvmArgumentProviders().add(nonInputProperties);
             test.getExtensions().add("nonInputProperties", nonInputProperties);
 
-            test.setWorkingDir(project.file(project.getBuildDir() + "/testrun/" + test.getName()));
+            test.setWorkingDir(project.file(project.getBuildDir() + "/testrun/" + test.getName().replace("#", "_")));
             test.setMaxParallelForks(Integer.parseInt(System.getProperty("tests.jvms", BuildParams.getDefaultParallel().toString())));
 
             test.exclude("**/*$*.class");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java

@@ -32,7 +32,7 @@ public void registerTestArtifactFromSourceSet(SourceSet sourceSet) {
         JavaPluginExtension javaPluginExtension = project.getExtensions().getByType(JavaPluginExtension.class);
         javaPluginExtension.registerFeature(name + "Artifacts", featureSpec -> {
             featureSpec.usingSourceSet(sourceSet);
-            featureSpec.capability("org.elasticsearch.gradle", project.getName() + "-" + name + "-artifacts", "1.0");
+            featureSpec.capability("org.elasticsearch.gradle", project.getName() + "-test-artifacts", "1.0");
             // This feature is only used internally in the
             // elasticsearch build so we do not need any publication.
             featureSpec.disablePublication();

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -13,6 +13,8 @@
 import org.elasticsearch.gradle.Architecture;
 import org.elasticsearch.gradle.DistributionDownloadPlugin;
 import org.elasticsearch.gradle.ElasticsearchDistribution;
+import org.elasticsearch.gradle.ElasticsearchDistributionType;
+import org.elasticsearch.gradle.Version;
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes;
 import org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin;
@@ -58,6 +60,8 @@ public class RestTestBasePlugin implements Plugin<Project> {
     private static final String TESTS_RUNTIME_JAVA_SYSPROP = "tests.runtime.java";
     private static final String DEFAULT_DISTRIBUTION_SYSPROP = "tests.default.distribution";
     private static final String INTEG_TEST_DISTRIBUTION_SYSPROP = "tests.integ-test.distribution";
+    private static final String BWC_SNAPSHOT_DISTRIBUTION_SYSPROP_PREFIX = "tests.snapshot.distribution.";
+    private static final String BWC_RELEASED_DISTRIBUTION_SYSPROP_PREFIX = "tests.release.distribution.";
     private static final String TESTS_CLUSTER_MODULES_PATH_SYSPROP = "tests.cluster.modules.path";
     private static final String TESTS_CLUSTER_PLUGINS_PATH_SYSPROP = "tests.cluster.plugins.path";
     private static final String DEFAULT_REST_INTEG_TEST_DISTRO = "default_distro";
@@ -79,16 +83,17 @@ public void apply(Project project) {
         project.getPluginManager().apply(InternalDistributionDownloadPlugin.class);
 
         // Register integ-test and default distributions
-        NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
-        ElasticsearchDistribution defaultDistro = distributions.create(DEFAULT_REST_INTEG_TEST_DISTRO, distro -> {
-            distro.setVersion(VersionProperties.getElasticsearch());
-            distro.setArchitecture(Architecture.current());
-        });
-        ElasticsearchDistribution integTestDistro = distributions.create(INTEG_TEST_REST_INTEG_TEST_DISTRO, distro -> {
-            distro.setVersion(VersionProperties.getElasticsearch());
-            distro.setArchitecture(Architecture.current());
-            distro.setType(ElasticsearchDistributionTypes.INTEG_TEST_ZIP);
-        });
+        ElasticsearchDistribution defaultDistro = createDistribution(
+            project,
+            DEFAULT_REST_INTEG_TEST_DISTRO,
+            VersionProperties.getElasticsearch()
+        );
+        ElasticsearchDistribution integTestDistro = createDistribution(
+            project,
+            INTEG_TEST_REST_INTEG_TEST_DISTRO,
+            VersionProperties.getElasticsearch(),
+            ElasticsearchDistributionTypes.INTEG_TEST_ZIP
+        );
 
         // Create configures for module and plugin dependencies
         Configuration modulesConfiguration = createPluginConfiguration(project, MODULES_CONFIGURATION, true, false);
@@ -151,13 +156,62 @@ public Void call(Object... args) {
                     return null;
                 }
             });
+
+            // Add `usesBwcDistribution(version)` extension method to test tasks to indicate they require a BWC distribution
+            task.getExtensions().getExtraProperties().set("usesBwcDistribution", new Closure<Void>(task) {
+                @Override
+                public Void call(Object... args) {
+                    if (args.length != 1 && args[0] instanceof Version == false) {
+                        throw new IllegalArgumentException("Expected exactly one argument of type org.elasticsearch.gradle.Version");
+                    }
+
+                    Version version = (Version) args[0];
+                    boolean isReleased = BuildParams.getBwcVersions().unreleasedInfo(version) == null;
+                    String versionString = version.toString();
+                    ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
+
+                    task.dependsOn(bwcDistro);
+                    registerDistributionInputs(task, bwcDistro);
+
+                    nonInputSystemProperties.systemProperty(
+                        (isReleased ? BWC_RELEASED_DISTRIBUTION_SYSPROP_PREFIX : BWC_SNAPSHOT_DISTRIBUTION_SYSPROP_PREFIX) + versionString,
+                        providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath())
+                    );
+
+                    if (version.before(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion())) {
+                        // If we are upgrade testing older versions we also need to upgrade to 7.last
+                        this.call(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion());
+                    }
+                    return null;
+                }
+            });
         });
 
         project.getTasks()
             .named(JavaBasePlugin.CHECK_TASK_NAME)
             .configure(check -> check.dependsOn(project.getTasks().withType(StandaloneRestIntegTestTask.class)));
     }
 
+    private ElasticsearchDistribution createDistribution(Project project, String name, String version) {
+        return createDistribution(project, name, version, null);
+    }
+
+    private ElasticsearchDistribution createDistribution(Project project, String name, String version, ElasticsearchDistributionType type) {
+        NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
+        ElasticsearchDistribution maybeDistro = distributions.findByName(name);
+        if (maybeDistro == null) {
+            return distributions.create(name, distro -> {
+                distro.setVersion(version);
+                distro.setArchitecture(Architecture.current());
+                if (type != null) {
+                    distro.setType(type);
+                }
+            });
+        } else {
+            return maybeDistro;
+        }
+    }
+
     private FileTree getDistributionFiles(ElasticsearchDistribution distribution, Action<PatternFilterable> patternFilter) {
         return distribution.getExtracted().getAsFileTree().matching(patternFilter);
     }

qa/full-cluster-restart/build.gradle

@@ -6,64 +6,20 @@
  * Side Public License, v 1.
  */
 
-
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
-apply plugin: 'elasticsearch.internal-testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.internal-test-artifact'
+apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.internal-test-artifact-base'
 apply plugin: 'elasticsearch.bwc-test'
 
-BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
-  def baseCluster = testClusters.register(baseName) {
-      if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) {
-        // When testing older versions we have to first upgrade to 7.last
-        versions = [bwcVersion.toString(), BuildParams.bwcVersions.minimumWireCompatibleVersion.toString(), project.version]
-      } else {
-        versions = [bwcVersion.toString(), project.version]
-      }
-      numberOfNodes = 2
-      // some tests rely on the translog not being flushed
-      setting 'indices.memory.shard_inactive_time', '60m'
-      setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-      setting 'xpack.security.enabled', 'false'
-      requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
-  }
-
-  tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) {
-    useCluster baseCluster
-    mustRunAfter("precommit")
-    doFirst {
-      delete("${buildDir}/cluster/shared/repo/${baseName}")
-    }
-
-    systemProperty 'tests.is_old_cluster', 'true'
-  }
-
-  tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
-    useCluster baseCluster
-    dependsOn "${baseName}#oldClusterTest"
-    doFirst {
-      baseCluster.get().goToNextVersion()
-      if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) {
-        // When doing a full cluster restart of older versions we actually have to upgrade twice. First to 7.last, then to the current version.
-        baseCluster.get().goToNextVersion()
-      }
-    }
-    systemProperty 'tests.is_old_cluster', 'false'
-  }
-
-  String oldVersion = bwcVersion.toString().minus("-SNAPSHOT")
-  tasks.matching { it.name.startsWith(baseName) && it.name.endsWith("ClusterTest") }.configureEach {
-    it.systemProperty 'tests.old_cluster_version', oldVersion
-    it.systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-    it.nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
-    it.nonInputProperties.systemProperty('tests.clustername', baseName)
-  }
+testArtifacts {
+  registerTestArtifactFromSourceSet(sourceSets.javaRestTest)
+}
 
-  tasks.register(bwcTaskName(bwcVersion)) {
-    dependsOn tasks.named("${baseName}#upgradedClusterTest")
+BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
+  tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
+    usesBwcDistribution(bwcVersion)
+    systemProperty("tests.old_cluster_version", bwcVersion)
   }
 }

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java → qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.upgrades;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.settings.RestClusterGetSettingsResponse;
@@ -28,13 +30,21 @@
 import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
 import org.elasticsearch.test.NotEqualMessageBuilder;
 import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.elasticsearch.transport.Compression;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -44,7 +54,6 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -80,13 +89,41 @@
  * version is started with the same data directories and then this is rerun
  * with {@code tests.is_old_cluster} set to {@code false}.
  */
-public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
+public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCase {
+
+    private static TemporaryFolder repoDirectory = new TemporaryFolder();
+
+    protected static LocalClusterConfigProvider clusterConfig = c -> {};
+
+    private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(getOldClusterTestVersion())
+        .nodes(2)
+        .setting("path.repo", () -> repoDirectory.getRoot().getPath())
+        .setting("xpack.security.enabled", "false")
+        // some tests rely on the translog not being flushed
+        .setting("indices.memory.shard_inactive_time", "60m")
+        .apply(() -> clusterConfig)
+        .feature(FeatureFlag.TIME_SERIES_MODE)
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster);
 
     private String index;
 
+    public FullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
+    @Override
+    protected ElasticsearchCluster getUpgradeCluster() {
+        return cluster;
+    }
+
     @Before
     public void setIndex() {
-        index = getTestName().toLowerCase(Locale.ROOT);
+        index = getRootTestName();
     }
 
     public void testSearch() throws Exception {
@@ -1051,7 +1088,7 @@ public void testSnapshotRestore() throws IOException {
                 repoConfig.startObject("settings");
                 {
                     repoConfig.field("compress", randomBoolean());
-                    repoConfig.field("location", System.getProperty("tests.path.repo"));
+                    repoConfig.field("location", repoDirectory.getRoot().getPath());
                 }
                 repoConfig.endObject();
             }
@@ -1725,7 +1762,7 @@ public void testEnableSoftDeletesOnRestore() throws Exception {
                 repoConfig.startObject("settings");
                 {
                     repoConfig.field("compress", randomBoolean());
-                    repoConfig.field("location", System.getProperty("tests.path.repo"));
+                    repoConfig.field("location", repoDirectory.getRoot().getPath());
                 }
                 repoConfig.endObject();
             }
@@ -1785,7 +1822,7 @@ public void testForbidDisableSoftDeletesOnRestore() throws Exception {
                 repoConfig.startObject("settings");
                 {
                     repoConfig.field("compress", randomBoolean());
-                    repoConfig.field("location", System.getProperty("tests.path.repo"));
+                    repoConfig.field("location", repoDirectory.getRoot().getPath());
                 }
                 repoConfig.endObject();
             }

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartTestOrdering.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.TestMethodAndParams;
+
+import java.util.Comparator;
+
+public class FullClusterRestartTestOrdering implements Comparator<TestMethodAndParams> {
+    @Override
+    public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
+        return Integer.compare(getOrdinal(o1), getOrdinal(o2));
+    }
+
+    private int getOrdinal(TestMethodAndParams t) {
+        return ((FullClusterRestartUpgradeStatus) t.getInstanceArguments().get(0)).ordinal();
+    }
+}

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartUpgradeStatus.java

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+public enum FullClusterRestartUpgradeStatus {
+    OLD,
+    UPGRADED
+}