From 06d1d7b1f3aed1cfb7a82d801a4665c69890580d Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 26 Nov 2018 17:25:01 +0200 Subject: [PATCH 01/17] Add a check to detect test classes that are not run Covers situtatuins as descrobed in #35435 where test classes don't have tasks that include them. --- .../precommit/TestingConventionsTasks.java | 132 +++++++++++++----- 1 file changed, 97 insertions(+), 35 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java index 1e73fa7cd0f24..5163f4ea594b1 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java @@ -20,15 +20,19 @@ import org.elasticsearch.gradle.tool.Boilerplate; import org.gradle.api.DefaultTask; +import org.gradle.api.Task; import org.gradle.api.file.FileCollection; +import org.gradle.api.file.FileTree; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.SkipWhenEmpty; import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.util.PatternFilterable; import java.io.File; import java.io.IOException; import java.lang.annotation.Annotation; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.net.MalformedURLException; @@ -40,8 +44,10 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -57,7 +63,7 @@ public class TestingConventionsTasks extends DefaultTask { */ private Boolean activeTestsExists; - private List testClassNames; + private Map testClassNames; public TestingConventionsTasks() { setDescription("Tests various testing conventions"); @@ -68,56 +74,113 @@ public TestingConventionsTasks() { @TaskAction public void doCheck() throws IOException { activeTestsExists = false; - final List problems; + final Optional problems; try (URLClassLoader isolatedClassLoader = new URLClassLoader( getTestsClassPath().getFiles().stream().map(this::fileToUrl).toArray(URL[]::new) )) { - List> classes = getTestClassNames().stream() - .map(name -> loadClassWithoutInitializing(name, isolatedClassLoader)) - .collect(Collectors.toList()); - Predicate> isStaticClass = clazz -> Modifier.isStatic(clazz.getModifiers()); Predicate> isPublicClass = clazz -> Modifier.isPublic(clazz.getModifiers()); - Predicate> implementsNamingConvention = clazz -> clazz.getName().endsWith(TEST_CLASS_SUFIX) || - clazz.getName().endsWith(INTEG_TEST_CLASS_SUFIX); + Predicate> implementsNamingConvention = clazz -> + clazz.getName().endsWith(TEST_CLASS_SUFIX) || + clazz.getName().endsWith(INTEG_TEST_CLASS_SUFIX); + + Map> classes = getTestClassNames().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getValue, + entry -> loadClassWithoutInitializing(entry.getKey(), isolatedClassLoader)) + ); - problems = Stream.concat( + FileTree allClassFiles = getProject().files( + classes.values().stream() + .filter(isStaticClass.negate()) + .filter(isPublicClass) + .filter(implementsNamingConvention) + .map(clazz -> testClassNames.get(clazz.getName())) + .collect(Collectors.toList()) + ).getAsFileTree(); + + final Map> filesPerTask = getProject().getTasks().withType(getRandomizedTestingTask()).stream() + .map(each -> (Task) each) + .collect(Collectors.toMap( + Task::getName, + task -> allClassFiles.matching(getRandomizedTestingPatternSet(task)).getFiles() + )); + + problems = collectProblems( checkNoneExists( "Test classes implemented by inner classes will not run", - classes.stream() + classes.values().stream() .filter(isStaticClass) .filter(implementsNamingConvention.or(this::seemsLikeATest)) - ).stream(), + ), checkNoneExists( "Seem like test classes but don't match naming convention", - classes.stream() + classes.values().stream() .filter(isStaticClass.negate()) .filter(isPublicClass) .filter(this::seemsLikeATest) .filter(implementsNamingConvention.negate()) - ).stream() - ).collect(Collectors.toList()); + ), + checkNoneExists( + "Test classes are not included in any task", + allClassFiles.getFiles().stream() + .filter(testFile -> + filesPerTask.values().stream() + .anyMatch(fileSet -> fileSet.contains(testFile)) == false + ) + .map(classes::get) + ) + ); } - if (problems.isEmpty()) { + if (problems.isPresent()) { + throw new IllegalStateException("Testing conventions are not honored:\n" + problems.get()); + } else { getSuccessMarker().getParentFile().mkdirs(); Files.write(getSuccessMarker().toPath(), new byte[]{}, StandardOpenOption.CREATE); - } else { - problems.forEach(getProject().getLogger()::error); - throw new IllegalStateException("Testing conventions are not honored"); + } + } + + private Optional collectProblems(String... problems) { + return Stream.of(problems) + .filter(String::isBlank) + .map(each -> each + "\n") + .reduce(String::concat) + .map(String::trim); + } + + private PatternFilterable getRandomizedTestingPatternSet(Task task) { + try { + if (getRandomizedTestingTask().isAssignableFrom(task.getClass()) == false) { + throw new IllegalStateException("Expected " + task + " to be RandomizedTestingTask but it was " + task.getClass()); + } + Method getPatternSet = task.getClass().getMethod("getPatternSet"); + return (PatternFilterable) getPatternSet.invoke(task); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("Expecte task to have a `patternSet` " + task, e); + } catch (IllegalAccessException|InvocationTargetException e) { + throw new IllegalStateException("Failed to get pattern set from task" + task, e); + } + } + + private Class getRandomizedTestingTask() { + try { + return (Class) Class.forName("com.carrotsearch.gradle.junit4.RandomizedTestingTask"); + } catch (ClassNotFoundException|ClassCastException e) { + throw new IllegalStateException("Failed to load randomized testing class", e); } } @Input @SkipWhenEmpty - public List getTestClassNames() { + public Map getTestClassNames() { if (testClassNames == null) { testClassNames = Boilerplate.getJavaSourceSets(getProject()).getByName("test").getOutput().getClassesDirs() .getFiles().stream() .filter(File::exists) - .flatMap(testRoot -> walkPathAndLoadClasses(testRoot).stream()) - .collect(Collectors.toList()); + .flatMap(testRoot -> walkPathAndLoadClasses(testRoot).entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } return testClassNames; } @@ -127,16 +190,15 @@ public File getSuccessMarker() { return new File(getProject().getBuildDir(), "markers/" + getName()); } - private List checkNoneExists(String message, Stream> stream) { - List problems = new ArrayList<>(); - List> entries = stream.collect(Collectors.toList()); - if (entries.isEmpty() == false) { - problems.add(message + ":"); - entries.stream() - .map(each -> " * " + each.getName()) - .forEach(problems::add); + private String checkNoneExists(String message, Stream> stream) { + String problem = stream + .map(each -> " * " + each.getName()) + .collect(Collectors.joining("\n")); + if (problem.isEmpty() == false) { + return message + ":\n" + problem; + } else{ + return ""; } - return problems; } private boolean seemsLikeATest(Class clazz) { @@ -197,8 +259,8 @@ private FileCollection getTestsClassPath() { ); } - private List walkPathAndLoadClasses(File testRoot) { - List classes = new ArrayList<>(); + private Map walkPathAndLoadClasses(File testRoot) { + Map classes = new HashMap<>(); try { Files.walkFileTree(testRoot.toPath(), new FileVisitor() { private String packageName; @@ -227,7 +289,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO String filename = file.getFileName().toString(); if (filename.endsWith(".class")) { String className = filename.substring(0, filename.length() - ".class".length()); - classes.add(packageName + className); + classes.put(packageName + className, file.toFile()); } return FileVisitResult.CONTINUE; } From e0dfc499cc223570e1a48d362959d782b7a9b927 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 26 Nov 2018 22:06:59 +0200 Subject: [PATCH 02/17] run all tests as build tools u You are currently bisecting, started from branch 'master'. --- buildSrc/build.gradle | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 793a6540f383e..1734f3016b869 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -147,10 +147,8 @@ if (project == rootProject) { mavenLocal() } } - test { - include "**/*Tests.class" - exclude "**/*IT.class" - } + // only run tests as build-tools + test.enabled = false } /***************************************************************************** @@ -178,7 +176,11 @@ if (project != rootProject) { // tests can't be run with randomized test runner // it's fine as we run them as part of :buildSrc - test.enabled = false + test { + include "**/*Tests.class" + exclude "**/*IT.class" + } + // This can't be an RandomizedTestingTask because we can't yet reference it task integTest(type: Test) { // integration test requires the local testing repo for example plugin builds dependsOn project.rootProject.allprojects.collect { From 78bb9142fb4188c91216a0b684bd2c707bd7d6ef Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 26 Nov 2018 22:07:33 +0200 Subject: [PATCH 03/17] disable testing conventions when no test are ran So the check doesn't complain that the tests are not ran --- x-pack/qa/third-party/hipchat/build.gradle | 1 + x-pack/qa/third-party/jira/build.gradle | 1 + x-pack/qa/third-party/pagerduty/build.gradle | 3 ++- x-pack/qa/third-party/slack/build.gradle | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/qa/third-party/hipchat/build.gradle b/x-pack/qa/third-party/hipchat/build.gradle index 2b2ee7fcbbf87..d72bb778e52c8 100644 --- a/x-pack/qa/third-party/hipchat/build.gradle +++ b/x-pack/qa/third-party/hipchat/build.gradle @@ -29,4 +29,5 @@ integTestCluster { if (!integrationAccount && !userAccount && !v1Account) { integTest.enabled = false + testingConventions.enabled = false } diff --git a/x-pack/qa/third-party/jira/build.gradle b/x-pack/qa/third-party/jira/build.gradle index eb53ccf642039..43667300a3383 100644 --- a/x-pack/qa/third-party/jira/build.gradle +++ b/x-pack/qa/third-party/jira/build.gradle @@ -47,6 +47,7 @@ task cleanJira(type: DefaultTask) { // require network access for this one, exit early instead of starting up the cluster if we dont have network if (!jiraUrl && !jiraUser && !jiraPassword && !jiraProject) { integTest.enabled = false + testingConventions.enabled = false } else { integTestRunner.finalizedBy cleanJira } diff --git a/x-pack/qa/third-party/pagerduty/build.gradle b/x-pack/qa/third-party/pagerduty/build.gradle index 12758989d0f21..9013d8c281538 100644 --- a/x-pack/qa/third-party/pagerduty/build.gradle +++ b/x-pack/qa/third-party/pagerduty/build.gradle @@ -18,5 +18,6 @@ integTestCluster { } if (!pagerDutyServiceKey) { - integTest.enabled = false + integTest.enabled = false + testingConventions.enabled = false } diff --git a/x-pack/qa/third-party/slack/build.gradle b/x-pack/qa/third-party/slack/build.gradle index f1bcd98cff694..03537966b29ff 100644 --- a/x-pack/qa/third-party/slack/build.gradle +++ b/x-pack/qa/third-party/slack/build.gradle @@ -22,4 +22,5 @@ integTestCluster { if (!slackUrl) { integTest.enabled = false + testingConventions.enabled = false } From 3231a7473a8b753b8d1c175a478c80be7a6dac75 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 26 Nov 2018 22:17:49 +0200 Subject: [PATCH 04/17] Disable testing conventions when tests disabled --- .../elasticsearch/gradle/precommit/TestingConventionsTasks.java | 2 +- x-pack/qa/kerberos-tests/build.gradle | 1 + x-pack/qa/openldap-tests/build.gradle | 1 + x-pack/qa/saml-idp-tests/build.gradle | 1 + x-pack/qa/third-party/active-directory/build.gradle | 2 -- 5 files changed, 4 insertions(+), 3 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java index 5163f4ea594b1..e47591ce5d90b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java @@ -106,7 +106,7 @@ public void doCheck() throws IOException { Task::getName, task -> allClassFiles.matching(getRandomizedTestingPatternSet(task)).getFiles() )); - + problems = collectProblems( checkNoneExists( "Test classes implemented by inner classes will not run", diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle index 5327fca1a6841..a59becbfe6b54 100644 --- a/x-pack/qa/kerberos-tests/build.gradle +++ b/x-pack/qa/kerberos-tests/build.gradle @@ -133,6 +133,7 @@ integTestRunner { if (project.rootProject.vagrantSupported == false) { integTest.enabled = false + testingConventions.enabled = false } else { project.sourceSets.test.output.dir(generatedResources) integTestCluster.dependsOn krb5AddPrincipals, krb5kdcFixture, copyKeytabToGeneratedResources diff --git a/x-pack/qa/openldap-tests/build.gradle b/x-pack/qa/openldap-tests/build.gradle index bb9a979928978..9e4c7ec2cebfe 100644 --- a/x-pack/qa/openldap-tests/build.gradle +++ b/x-pack/qa/openldap-tests/build.gradle @@ -27,6 +27,7 @@ if (project.rootProject.vagrantSupported) { test.finalizedBy idpFixtureProject.halt } else { test.enabled = false + testingConventions.enabled = false } namingConventions { diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle index 8e6672f21e9d5..00500c448d9c5 100644 --- a/x-pack/qa/saml-idp-tests/build.gradle +++ b/x-pack/qa/saml-idp-tests/build.gradle @@ -28,6 +28,7 @@ if (project.rootProject.vagrantSupported) { integTest.finalizedBy idpFixtureProject.halt } else { integTest.enabled = false + testingConventions.enabled = false } integTestCluster { diff --git a/x-pack/qa/third-party/active-directory/build.gradle b/x-pack/qa/third-party/active-directory/build.gradle index 4acebbda9a5b6..6ce8541fc431f 100644 --- a/x-pack/qa/third-party/active-directory/build.gradle +++ b/x-pack/qa/third-party/active-directory/build.gradle @@ -29,5 +29,3 @@ test { include '**/*Tests.class' } -// these are just tests, no need to audit -thirdPartyAudit.enabled = false \ No newline at end of file From 69dda277b97f19dcc50409311d0bd7e218163063 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 26 Nov 2018 22:20:45 +0200 Subject: [PATCH 05/17] Do more auto configuration for additional testing tasks --- .../org/elasticsearch/gradle/BuildPlugin.groovy | 12 ++++++++++++ .../gradle/test/RestIntegTestTask.groovy | 1 - plugins/repository-hdfs/build.gradle | 1 + plugins/repository-s3/build.gradle | 3 --- server/build.gradle | 4 ---- x-pack/plugin/ccr/build.gradle | 5 ----- x-pack/plugin/ml/build.gradle | 5 ----- x-pack/plugin/monitoring/build.gradle | 4 ---- x-pack/plugin/upgrade/build.gradle | 4 ---- 9 files changed, 13 insertions(+), 26 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 92047aca066d1..0e493eb37e8b2 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -777,6 +777,18 @@ class BuildPlugin implements Plugin { onNonEmptyWorkDirectory 'wipe' leaveTemporary true + if (name != "test") { + project.tasks.matching { it.name == "test"}.all { testTask -> + task.testClassesDirs = testTask.testClassesDirs + task.classpath = testTask.classpath + task.shouldRunAfter testTask + // no loose ends: check has to depend on all test tasks + project.tasks.matching {it.name == "check"}.all { + dependsOn(task) + } + } + } + // TODO: why are we not passing maxmemory to junit4? jvmArg '-Xmx' + System.getProperty('tests.heap.size', '512m') jvmArg '-Xms' + System.getProperty('tests.heap.size', '512m') diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy index 689cf5bf2ed2c..a17356c3daa44 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy @@ -61,7 +61,6 @@ public class RestIntegTestTask extends DefaultTask { clusterInit = project.tasks.create(name: "${name}Cluster#init", dependsOn: project.testClasses) runner.dependsOn(clusterInit) runner.classpath = project.sourceSets.test.runtimeClasspath - runner.testClassesDirs = project.sourceSets.test.output.classesDirs clusterConfig = project.extensions.create("${name}Cluster", ClusterConfiguration.class, project) // override/add more for rest tests diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 557dcaa5faedc..0f18c6af2d923 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -301,6 +301,7 @@ if (secureFixtureSupported) { // Security tests unsupported. Don't run these tests. integTestSecure.enabled = false integTestSecureHa.enabled = false + testingConventions.enabled = false } thirdPartyAudit.excludes = [ diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 3e6c7a1318df3..b74fbbe54f403 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -73,10 +73,7 @@ task testRepositoryCreds(type: RandomizedTestingTask) { include '**/RepositoryCredentialsTests.class' include '**/S3BlobStoreRepositoryTests.class' systemProperty 'es.allow_insecure_settings', 'true' - classpath = tasks.test.classpath - testClassesDirs = tasks.test.testClassesDirs } -project.check.dependsOn(testRepositoryCreds) test { // these are tested explicitly in separate test tasks diff --git a/server/build.gradle b/server/build.gradle index 39579bed28866..647362ef3c0a1 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -320,10 +320,6 @@ if (isEclipse == false || project.path == ":server-tests") { group: JavaBasePlugin.VERIFICATION_GROUP, description: 'Multi-node tests', dependsOn: test.dependsOn) { - classpath = project.test.classpath - testClassesDirs = project.test.testClassesDirs include '**/*IT.class' } - check.dependsOn integTest - integTest.mustRunAfter test } diff --git a/x-pack/plugin/ccr/build.gradle b/x-pack/plugin/ccr/build.gradle index fe5341ef99372..f70e15bac9fb7 100644 --- a/x-pack/plugin/ccr/build.gradle +++ b/x-pack/plugin/ccr/build.gradle @@ -24,15 +24,10 @@ task internalClusterTest(type: RandomizedTestingTask, group: JavaBasePlugin.VERIFICATION_GROUP, description: 'Java fantasy integration tests', dependsOn: test.dependsOn) { - classpath = project.test.classpath - testClassesDirs = project.test.testClassesDirs include '**/*IT.class' systemProperty 'es.set.netty.runtime.available.processors', 'false' } -check.dependsOn internalClusterTest -internalClusterTest.mustRunAfter test - // add all sub-projects of the qa sub-project gradle.projectsEvaluated { project.subprojects diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle index f3a6dc8b7a49f..ab95b02ffdbc8 100644 --- a/x-pack/plugin/ml/build.gradle +++ b/x-pack/plugin/ml/build.gradle @@ -98,15 +98,10 @@ task internalClusterTest(type: RandomizedTestingTask, group: JavaBasePlugin.VERIFICATION_GROUP, description: 'Multi-node tests', dependsOn: test.dependsOn) { - classpath = project.test.classpath - testClassesDirs = project.test.testClassesDirs include '**/*IT.class' systemProperty 'es.set.netty.runtime.available.processors', 'false' } -check.dependsOn internalClusterTest -internalClusterTest.mustRunAfter test - // add all sub-projects of the qa sub-project gradle.projectsEvaluated { project.subprojects diff --git a/x-pack/plugin/monitoring/build.gradle b/x-pack/plugin/monitoring/build.gradle index 54df68e769c39..08a73d1fa01b8 100644 --- a/x-pack/plugin/monitoring/build.gradle +++ b/x-pack/plugin/monitoring/build.gradle @@ -60,13 +60,9 @@ task internalClusterTest(type: RandomizedTestingTask, group: JavaBasePlugin.VERIFICATION_GROUP, description: 'Multi-node tests', dependsOn: test.dependsOn) { - classpath = project.test.classpath - testClassesDirs = project.test.testClassesDirs include '**/*IT.class' systemProperty 'es.set.netty.runtime.available.processors', 'false' } -check.dependsOn internalClusterTest -internalClusterTest.mustRunAfter test // also add an "alias" task to make typing on the command line easier task icTest { task icTest { diff --git a/x-pack/plugin/upgrade/build.gradle b/x-pack/plugin/upgrade/build.gradle index 309962fa487ff..4ee315af9107d 100644 --- a/x-pack/plugin/upgrade/build.gradle +++ b/x-pack/plugin/upgrade/build.gradle @@ -33,13 +33,9 @@ task internalClusterTest(type: RandomizedTestingTask, group: JavaBasePlugin.VERIFICATION_GROUP, description: 'Multi-node tests', dependsOn: test.dependsOn) { - classpath = project.test.classpath - testClassesDirs = project.test.testClassesDirs include '**/*IT.class' systemProperty 'es.set.netty.runtime.available.processors', 'false' } -check.dependsOn internalClusterTest -internalClusterTest.mustRunAfter test // also add an "alias" task to make typing on the command line easier task icTest { From b8ca5efd11e5dfd832da2ab316f6c23439917ce8 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 26 Nov 2018 22:21:12 +0200 Subject: [PATCH 06/17] Run integ tests on test/framework --- test/framework/build.gradle | 6 ++++++ .../elasticsearch/test/disruption/NetworkDisruptionIT.java | 1 + .../org/elasticsearch/test/test/SuiteScopeClusterIT.java | 7 ------- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 12653cc6489ae..0d5ee676c79b9 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -16,6 +16,8 @@ * specific language governing permissions and limitations * under the License. */ +import com.carrotsearch.gradle.junit4.RandomizedTestingTask; + dependencies { compile "org.elasticsearch.client:elasticsearch-rest-client:${version}" compile "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}" @@ -73,3 +75,7 @@ test.configure { systemProperty 'tests.gradle_wire_compat_versions', bwcVersions.wireCompatible.join(',') systemProperty 'tests.gradle_unreleased_versions', bwcVersions.unreleased.join(',') } + +task integTest(type: RandomizedTestingTask) { + include "**/*IT.class" +} diff --git a/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java b/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java index 65850022aaaa5..8c4b6e557d3e7 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java +++ b/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java @@ -40,6 +40,7 @@ protected Collection> nodePlugins() { return Arrays.asList(MockTransportService.TestPlugin.class); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/35861") public void testNetworkPartitionWithNodeShutdown() throws IOException { internalCluster().ensureAtLeastNumDataNodes(2); String[] nodeNames = internalCluster().getNodeNames(); diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java b/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java index 4e0623cd13448..38e15878a984e 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java @@ -35,7 +35,6 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) public class SuiteScopeClusterIT extends ESIntegTestCase { private static int ITER = 0; - private static long[] SEQUENCE = new long[100]; private static Long CLUSTER_SEED = null; @Test @@ -44,14 +43,8 @@ public class SuiteScopeClusterIT extends ESIntegTestCase { public void testReproducible() throws IOException { if (ITER++ == 0) { CLUSTER_SEED = cluster().seed(); - for (int i = 0; i < SEQUENCE.length; i++) { - SEQUENCE[i] = randomLong(); - } } else { assertEquals(CLUSTER_SEED, Long.valueOf(cluster().seed())); - for (int i = 0; i < SEQUENCE.length; i++) { - assertThat(SEQUENCE[i], equalTo(randomLong())); - } } } From 97177ce596e675843f671aee48490126aa0a53b1 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 26 Nov 2018 22:21:39 +0200 Subject: [PATCH 07/17] remove redundant config --- .../org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index 28d18e9b876f5..038686247f4b6 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -129,7 +129,6 @@ public class PluginBuildPlugin extends BuildPlugin { RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class) integTest.mustRunAfter(project.precommit, project.test) project.integTestCluster.distribution = System.getProperty('tests.distribution', 'integ-test-zip') - project.check.dependsOn(integTest) } /** From 37c785e683ce78acfd1b4cb0f6c99c31eacffc91 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 26 Nov 2018 22:22:17 +0200 Subject: [PATCH 08/17] Add task to run test --- x-pack/plugin/core/build.gradle | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle index 647ebd53f1cd8..b68fbc1f8c5a7 100644 --- a/x-pack/plugin/core/build.gradle +++ b/x-pack/plugin/core/build.gradle @@ -4,6 +4,7 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.nio.file.StandardCopyOption +import com.carrotsearch.gradle.junit4.RandomizedTestingTask; apply plugin: 'elasticsearch.esplugin' apply plugin: 'nebula.maven-base-publish' @@ -136,5 +137,10 @@ thirdPartyAudit.excludes = [ // xpack modules are installed in real clusters as the meta plugin, so // installing them as individual plugins for integ tests doesn't make sense, -// so we disable integ tests and there are no integ tests in xpack core module +// so we disable integ tests integTest.enabled = false + +// There are some integ tests that don't require a cluster, we still want to run those +task internalClusterTest(type: RandomizedTestingTask) { + include "**/*IT.class" +} From a160688b98fb7e713919ef08516805aa7c96f805 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 26 Nov 2018 22:22:30 +0200 Subject: [PATCH 09/17] Add a check to assert that all tests are executed --- .../precommit/TestingConventionsTasks.java | 71 +++++++++++++++---- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java index e47591ce5d90b..ff47d351213bc 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java @@ -27,6 +27,7 @@ import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.SkipWhenEmpty; import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.testing.Test; import org.gradle.api.tasks.util.PatternFilterable; import java.io.File; @@ -91,7 +92,7 @@ public void doCheck() throws IOException { entry -> loadClassWithoutInitializing(entry.getKey(), isolatedClassLoader)) ); - FileTree allClassFiles = getProject().files( + FileTree allTestClassFiles = getProject().files( classes.values().stream() .filter(isStaticClass.negate()) .filter(isPublicClass) @@ -100,12 +101,8 @@ public void doCheck() throws IOException { .collect(Collectors.toList()) ).getAsFileTree(); - final Map> filesPerTask = getProject().getTasks().withType(getRandomizedTestingTask()).stream() - .map(each -> (Task) each) - .collect(Collectors.toMap( - Task::getName, - task -> allClassFiles.matching(getRandomizedTestingPatternSet(task)).getFiles() - )); + final Map> classFilesPerRandomizedTestingTask = classFilesPerRandomizedTestingTask(allTestClassFiles); + final Map> classFilesPerGradleTestTask = classFilesPerGradleTestTask(); problems = collectProblems( checkNoneExists( @@ -123,11 +120,17 @@ public void doCheck() throws IOException { .filter(implementsNamingConvention.negate()) ), checkNoneExists( - "Test classes are not included in any task", - allClassFiles.getFiles().stream() + "Test classes are not included in any enabled task (" + + Stream.concat( + classFilesPerRandomizedTestingTask.keySet().stream(), + classFilesPerGradleTestTask.keySet().stream() + ).collect(Collectors.joining(",")) + ")", + allTestClassFiles.getFiles().stream() .filter(testFile -> - filesPerTask.values().stream() - .anyMatch(fileSet -> fileSet.contains(testFile)) == false + classFilesPerRandomizedTestingTask.values().stream() + .anyMatch(fileSet -> fileSet.contains(testFile)) == false && + classFilesPerGradleTestTask.values().stream() + .anyMatch(fileSet -> fileSet.contains(testFile)) == false ) .map(classes::get) ) @@ -142,6 +145,7 @@ public void doCheck() throws IOException { } } + private Optional collectProblems(String... problems) { return Stream.of(problems) .filter(String::isBlank) @@ -150,24 +154,61 @@ private Optional collectProblems(String... problems) { .map(String::trim); } + + @Input + public Map> classFilesPerRandomizedTestingTask(FileTree testClassFiles) { + return Stream.concat( + getProject().getTasks().withType(getRandomizedTestingTask()).stream(), + // Look at sub-projects too. As sometimes tests are implemented in parent but ran in sub-projects against + // different configurations + getProject().getSubprojects().stream().flatMap(subproject -> + subproject.getTasks().withType(getRandomizedTestingTask()).stream() + ) + ) + .filter(Task::getEnabled) + .collect(Collectors.toMap( + Task::getPath, + task -> testClassFiles.matching(getRandomizedTestingPatternSet(task)).getFiles() + )); + } + + @Input + public Map> classFilesPerGradleTestTask() { + return Stream.concat( + getProject().getTasks().withType(Test.class).stream(), + getProject().getSubprojects().stream().flatMap(subproject -> + subproject.getTasks().withType(Test.class).stream() + ) + ) + .filter(Task::getEnabled) + .collect(Collectors.toMap( + Task::getPath, + task -> task.getCandidateClassFiles().getFiles() + )); + } + + @SuppressWarnings("unchecked") private PatternFilterable getRandomizedTestingPatternSet(Task task) { try { - if (getRandomizedTestingTask().isAssignableFrom(task.getClass()) == false) { - throw new IllegalStateException("Expected " + task + " to be RandomizedTestingTask but it was " + task.getClass()); + if ( + getRandomizedTestingTask().isAssignableFrom(task.getClass()) == false + ) { + throw new IllegalStateException("Expected " + task + " to be RandomizedTestingTask or Test but it was " + task.getClass()); } Method getPatternSet = task.getClass().getMethod("getPatternSet"); return (PatternFilterable) getPatternSet.invoke(task); } catch (NoSuchMethodException e) { throw new IllegalStateException("Expecte task to have a `patternSet` " + task, e); - } catch (IllegalAccessException|InvocationTargetException e) { + } catch (IllegalAccessException | InvocationTargetException e) { throw new IllegalStateException("Failed to get pattern set from task" + task, e); } } + @SuppressWarnings("unchecked") private Class getRandomizedTestingTask() { try { return (Class) Class.forName("com.carrotsearch.gradle.junit4.RandomizedTestingTask"); - } catch (ClassNotFoundException|ClassCastException e) { + } catch (ClassNotFoundException | ClassCastException e) { throw new IllegalStateException("Failed to load randomized testing class", e); } } From 7b2c2ba695caac1b5a1e5d8adf2beaeca648f855 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 26 Nov 2018 22:43:58 +0200 Subject: [PATCH 10/17] checkstyule --- .../java/org/elasticsearch/test/test/SuiteScopeClusterIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java b/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java index 38e15878a984e..8992b88a9377d 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java @@ -26,8 +26,6 @@ import java.io.IOException; -import static org.hamcrest.Matchers.equalTo; - /** * This test ensures that the cluster initializion for suite scope is not influencing * the tests random sequence due to initializtion using the same random instance. From 4017ed5d2a5e2a600292af4638ff5fcd941cadb7 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 26 Nov 2018 22:44:08 +0200 Subject: [PATCH 11/17] Assert that check depends on all test tasks --- build.gradle | 15 +++++++++++++++ .../org/elasticsearch/gradle/BuildPlugin.groovy | 2 +- .../precommit/TestingConventionsTasks.java | 16 ++++++++-------- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index 080119bfe57e1..9447029064cef 100644 --- a/build.gradle +++ b/build.gradle @@ -639,5 +639,20 @@ allprojects { } +if (gradle.startParameter.taskNames.contains("check")) { + gradle.taskGraph.whenReady { tg -> + allprojects { project -> + tasks.withType(RandomizedTestingTask.class) { each -> + if (tg.hasTask(each) == false) { + throw new IllegalStateException( + "Found that ${project.path}:check does not depend on testing task:${each.path}" + ) + } + } + } + } +} + + diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 0e493eb37e8b2..a2f816832f31d 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -770,7 +770,7 @@ class BuildPlugin implements Plugin { } static void applyCommonTestConfig(Project project) { - project.tasks.withType(RandomizedTestingTask) { + project.tasks.withType(RandomizedTestingTask) {task -> jvm "${project.runtimeJavaHome}/bin/java" parallelism System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel) ifNoTests System.getProperty('tests.ifNoTests', 'fail') diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java index ff47d351213bc..c5f8d50907329 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java @@ -47,7 +47,6 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.HashMap; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -75,7 +74,7 @@ public TestingConventionsTasks() { @TaskAction public void doCheck() throws IOException { activeTestsExists = false; - final Optional problems; + final String problems; try (URLClassLoader isolatedClassLoader = new URLClassLoader( getTestsClassPath().getFiles().stream().map(this::fileToUrl).toArray(URL[]::new) @@ -137,8 +136,9 @@ public void doCheck() throws IOException { ); } - if (problems.isPresent()) { - throw new IllegalStateException("Testing conventions are not honored:\n" + problems.get()); + if (problems.isEmpty()) { + getLogger().error(problems); + throw new IllegalStateException("Testing conventions are not honored"); } else { getSuccessMarker().getParentFile().mkdirs(); Files.write(getSuccessMarker().toPath(), new byte[]{}, StandardOpenOption.CREATE); @@ -146,12 +146,12 @@ public void doCheck() throws IOException { } - private Optional collectProblems(String... problems) { + private String collectProblems(String... problems) { return Stream.of(problems) - .filter(String::isBlank) + .map(String::trim) + .filter(String::isEmpty) .map(each -> each + "\n") - .reduce(String::concat) - .map(String::trim); + .collect(Collectors.joining()); } From e8922219c7145b64649a30f7bf169be905970eaa Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Fri, 30 Nov 2018 16:10:05 +0200 Subject: [PATCH 12/17] Remove unnecessary rule, doesn't work with in parallel --- .../gradle/precommit/FilePermissionsTaskTests.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/FilePermissionsTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/FilePermissionsTaskTests.java index 1b3593d52424f..574545c4361d6 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/FilePermissionsTaskTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/FilePermissionsTaskTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.gradle.precommit; import java.io.File; -import java.io.IOException; import java.nio.charset.Charset; import java.nio.file.Files; import java.util.List; @@ -32,12 +31,8 @@ import org.gradle.api.plugins.JavaPlugin; import org.gradle.testfixtures.ProjectBuilder; import org.junit.Assert; -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; public class FilePermissionsTaskTests extends GradleUnitTestCase { - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); public void testCheckPermissionsWhenAnExecutableFileExists() throws Exception { RandomizedTest.assumeFalse("Functionality is Unix specific", Os.isFamily(Os.FAMILY_WINDOWS)); @@ -93,11 +88,10 @@ public void testCheckPermissionsWhenNoExecutableFileExists() throws Exception { assertEquals("done", result.get(0)); file.delete(); - } - private Project createProject() throws IOException { - Project project = ProjectBuilder.builder().withProjectDir(temporaryFolder.newFolder()).build(); + private Project createProject() { + Project project = ProjectBuilder.builder().build(); project.getPlugins().apply(JavaPlugin.class); return project; } @@ -105,4 +99,5 @@ private Project createProject() throws IOException { private FilePermissionsTask createTask(Project project) { return project.getTasks().create("filePermissionsTask", FilePermissionsTask.class); } + } From 953f97e1aeb729592abc5e50cdafac3950e1b870 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Fri, 30 Nov 2018 16:24:49 +0200 Subject: [PATCH 13/17] Drop chack for dependencies --- build.gradle | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/build.gradle b/build.gradle index 9447029064cef..43eeeffa766e2 100644 --- a/build.gradle +++ b/build.gradle @@ -639,20 +639,4 @@ allprojects { } -if (gradle.startParameter.taskNames.contains("check")) { - gradle.taskGraph.whenReady { tg -> - allprojects { project -> - tasks.withType(RandomizedTestingTask.class) { each -> - if (tg.hasTask(each) == false) { - throw new IllegalStateException( - "Found that ${project.path}:check does not depend on testing task:${each.path}" - ) - } - } - } - } -} - - - From bbbe7f1a51f87d635f7ae3d31fbe1b8c0d9bbd72 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Fri, 30 Nov 2018 17:14:59 +0200 Subject: [PATCH 14/17] keep the setting of output directories --- .../org/elasticsearch/gradle/test/RestIntegTestTask.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy index a17356c3daa44..689cf5bf2ed2c 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy @@ -61,6 +61,7 @@ public class RestIntegTestTask extends DefaultTask { clusterInit = project.tasks.create(name: "${name}Cluster#init", dependsOn: project.testClasses) runner.dependsOn(clusterInit) runner.classpath = project.sourceSets.test.runtimeClasspath + runner.testClassesDirs = project.sourceSets.test.output.classesDirs clusterConfig = project.extensions.create("${name}Cluster", ClusterConfiguration.class, project) // override/add more for rest tests From 1905496d15d43a583478e39d43f3916713a3e22c Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 3 Dec 2018 11:30:40 +0200 Subject: [PATCH 15/17] PR review --- build.gradle | 1 + buildSrc/build.gradle | 3 +-- .../org/elasticsearch/gradle/BuildPlugin.groovy | 9 +++++---- .../gradle/precommit/TestingConventionsTasks.java | 15 ++++++++------- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/build.gradle b/build.gradle index 43eeeffa766e2..080119bfe57e1 100644 --- a/build.gradle +++ b/build.gradle @@ -640,3 +640,4 @@ allprojects { + diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 1734f3016b869..d907ceb81b512 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -174,12 +174,11 @@ if (project != rootProject) { jarHell.enabled = false thirdPartyAudit.enabled = false - // tests can't be run with randomized test runner - // it's fine as we run them as part of :buildSrc test { include "**/*Tests.class" exclude "**/*IT.class" } + // This can't be an RandomizedTestingTask because we can't yet reference it task integTest(type: Test) { // integration test requires the local testing repo for example plugin builds diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index a2f816832f31d..c164a56c9d6bc 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -777,17 +777,18 @@ class BuildPlugin implements Plugin { onNonEmptyWorkDirectory 'wipe' leaveTemporary true + // Make sure all test tasks are configured properly if (name != "test") { project.tasks.matching { it.name == "test"}.all { testTask -> task.testClassesDirs = testTask.testClassesDirs task.classpath = testTask.classpath task.shouldRunAfter testTask - // no loose ends: check has to depend on all test tasks - project.tasks.matching {it.name == "check"}.all { - dependsOn(task) - } } } + // no loose ends: check has to depend on all test tasks + project.tasks.matching {it.name == "check"}.all { + dependsOn(task) + } // TODO: why are we not passing maxmemory to junit4? jvmArg '-Xmx' + System.getProperty('tests.heap.size', '512m') diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java index c5f8d50907329..df7e71302152d 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java @@ -157,14 +157,15 @@ private String collectProblems(String... problems) { @Input public Map> classFilesPerRandomizedTestingTask(FileTree testClassFiles) { - return Stream.concat( - getProject().getTasks().withType(getRandomizedTestingTask()).stream(), - // Look at sub-projects too. As sometimes tests are implemented in parent but ran in sub-projects against - // different configurations - getProject().getSubprojects().stream().flatMap(subproject -> - subproject.getTasks().withType(getRandomizedTestingTask()).stream() + return + Stream.concat( + getProject().getTasks().withType(getRandomizedTestingTask()).stream(), + // Look at sub-projects too. As sometimes tests are implemented in parent but ran in sub-projects against + // different configurations + getProject().getSubprojects().stream().flatMap(subproject -> + subproject.getTasks().withType(getRandomizedTestingTask()).stream() + ) ) - ) .filter(Task::getEnabled) .collect(Collectors.toMap( Task::getPath, From d5f8dcc86e324131468a95cd79f0d111a220e61d Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 4 Dec 2018 10:41:12 +0200 Subject: [PATCH 16/17] revert SuiteScopeClusterIT --- .../org/elasticsearch/test/test/SuiteScopeClusterIT.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java b/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java index 8992b88a9377d..4e0623cd13448 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java @@ -26,6 +26,8 @@ import java.io.IOException; +import static org.hamcrest.Matchers.equalTo; + /** * This test ensures that the cluster initializion for suite scope is not influencing * the tests random sequence due to initializtion using the same random instance. @@ -33,6 +35,7 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) public class SuiteScopeClusterIT extends ESIntegTestCase { private static int ITER = 0; + private static long[] SEQUENCE = new long[100]; private static Long CLUSTER_SEED = null; @Test @@ -41,8 +44,14 @@ public class SuiteScopeClusterIT extends ESIntegTestCase { public void testReproducible() throws IOException { if (ITER++ == 0) { CLUSTER_SEED = cluster().seed(); + for (int i = 0; i < SEQUENCE.length; i++) { + SEQUENCE[i] = randomLong(); + } } else { assertEquals(CLUSTER_SEED, Long.valueOf(cluster().seed())); + for (int i = 0; i < SEQUENCE.length; i++) { + assertThat(SEQUENCE[i], equalTo(randomLong())); + } } } From 9c3d5b5a7dc95208c5e668bc71533005d3f3667f Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 4 Dec 2018 11:27:00 +0200 Subject: [PATCH 17/17] Mute failing tests --- .../org/elasticsearch/test/disruption/NetworkDisruptionIT.java | 3 ++- .../java/org/elasticsearch/test/test/SuiteScopeClusterIT.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java b/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java index 8c4b6e557d3e7..a75b260fa3b7c 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java +++ b/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.test.disruption; +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; @@ -34,13 +35,13 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; +@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/36205") public class NetworkDisruptionIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { return Arrays.asList(MockTransportService.TestPlugin.class); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/35861") public void testNetworkPartitionWithNodeShutdown() throws IOException { internalCluster().ensureAtLeastNumDataNodes(2); String[] nodeNames = internalCluster().getNodeNames(); diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java b/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java index 4e0623cd13448..cd3c3b2032331 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java @@ -41,6 +41,7 @@ public class SuiteScopeClusterIT extends ESIntegTestCase { @Test @SuppressForbidden(reason = "repeat is a feature here") @Repeat(iterations = 10, useConstantSeed = true) + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/36202") public void testReproducible() throws IOException { if (ITER++ == 0) { CLUSTER_SEED = cluster().seed();