From 1e9a4fb03e7d6d658dce0bfb8ce128d49e00f6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 21 Mar 2022 10:41:03 +0000 Subject: [PATCH 1/2] [SUREFIRE-2041] Add integration test for test order --- .../apache/maven/surefire/its/RunOrderIT.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java index 0032e8de74..05b5a47b25 100644 --- a/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java @@ -42,6 +42,8 @@ public class RunOrderIT private static final String[] TESTS_IN_REVERSE_ALPHABETICAL_ORDER = { "TC", "TB", "TA" }; + private static final String[] TESTS_IN_TEST_ORDER = { "TC", "TA", "TB" }; + // testing random is left as an exercise to the reader. Patches welcome @Test @@ -97,7 +99,7 @@ public void testRandomJUnit4SameSeed() } } } - + @Test public void testReverseAlphabeticalJUnit4() throws Exception @@ -150,6 +152,22 @@ public void testHourlyJUnit5() assertTestnamesAppearInSpecificOrder( validator, testnames ); } + @Test + public void testTestOrderJUnit4() + throws Exception + { + OutputValidator validator = executeWithTestOrder( "junit4" ); + assertTestnamesAppearInSpecificOrder( validator, TESTS_IN_TEST_ORDER ); + } + + @Test + public void testTestOrderJUnit5() + throws Exception + { + OutputValidator validator = executeWithTestOrder( "junit5" ); + assertTestnamesAppearInSpecificOrder( validator, TESTS_IN_TEST_ORDER ); + } + @Test public void testNonExistingRunOrderJUnit4() { @@ -198,6 +216,17 @@ private OutputValidator executeWithRandomOrder( String profile, long seed ) .verifyErrorFree( 3 ); } + private OutputValidator executeWithTestOrder( String profile ) + { + return unpack() + .activateProfile( profile ) + .forkMode( getForkMode() ) + .setTestToRun( "junit.runOrder.TestC,junit.runOrder.TestA,junit.runOrder.TestB" ) + .runOrder( "test" ) + .executeTest() + .verifyErrorFree( 3 ); + } + protected String getForkMode() { return "once"; From d4b496c9bde2c357c30fd1177d93aacb5e560695 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Mon, 21 Mar 2022 10:17:54 +0000 Subject: [PATCH 2/2] [SUREFIRE-2041] Add new test RunOrder --- .../api/booter/BaseProviderFactory.java | 9 +++- .../surefire/api/testset/ResolvedTest.java | 2 +- .../api/util/DefaultRunOrderCalculator.java | 21 ++++++++- .../maven/surefire/api/util/RunOrder.java | 5 ++- .../api/util/RunOrderCalculatorTest.java | 43 +++++++++++++++++-- 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/BaseProviderFactory.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/BaseProviderFactory.java index 1436e98dc6..35195ec217 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/BaseProviderFactory.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/booter/BaseProviderFactory.java @@ -25,8 +25,10 @@ import org.apache.maven.surefire.api.report.ReporterConfiguration; import org.apache.maven.surefire.api.report.ReporterFactory; import org.apache.maven.surefire.api.testset.DirectoryScannerParameters; +import org.apache.maven.surefire.api.testset.ResolvedTest; import org.apache.maven.surefire.api.testset.RunOrderParameters; import org.apache.maven.surefire.api.testset.TestArtifactInfo; +import org.apache.maven.surefire.api.testset.TestListResolver; import org.apache.maven.surefire.api.testset.TestRequest; import org.apache.maven.surefire.api.util.DefaultDirectoryScanner; import org.apache.maven.surefire.api.util.DefaultRunOrderCalculator; @@ -35,6 +37,7 @@ import org.apache.maven.surefire.api.util.RunOrderCalculator; import org.apache.maven.surefire.api.util.ScanResult; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; @@ -115,7 +118,11 @@ private int getThreadCount() @Override public RunOrderCalculator getRunOrderCalculator() { - return new DefaultRunOrderCalculator( runOrderParameters, getThreadCount() ); + TestListResolver testListResolver = testRequest.getTestListResolver(); + List resolvedTests = new ArrayList<>( testListResolver == null + ? emptyList() + : testListResolver.getIncludedPatterns() ); + return new DefaultRunOrderCalculator( runOrderParameters, getThreadCount(), resolvedTests ); } public void setReporterFactory( ReporterFactory reporterFactory ) diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/ResolvedTest.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/ResolvedTest.java index 24503581c0..0ae876fc57 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/ResolvedTest.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/ResolvedTest.java @@ -288,7 +288,7 @@ private boolean match( String testClassFile, String methodName ) return matchClass( testClassFile ) && matchMethod( methodName ); } - private boolean matchClass( String testClassFile ) + public boolean matchClass( String testClassFile ) { return classPattern == null || classMatcher.matchTestClassFile( testClassFile ); } diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java index 8ced06afa5..5ca7aa72a8 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java @@ -20,6 +20,7 @@ */ import org.apache.maven.surefire.api.runorder.RunEntryStatisticsMap; +import org.apache.maven.surefire.api.testset.ResolvedTest; import org.apache.maven.surefire.api.testset.RunOrderParameters; import java.util.ArrayList; @@ -29,6 +30,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Random; +import java.util.stream.IntStream; /** * Applies the final runorder of the tests @@ -42,17 +44,21 @@ public class DefaultRunOrderCalculator private final RunOrder[] runOrder; + private final List resolvedTests; + private final RunOrderParameters runOrderParameters; private final int threadCount; private final Random random; - public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount ) + public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount, + List resolvedTests ) { this.runOrderParameters = runOrderParameters; this.threadCount = threadCount; this.runOrder = runOrderParameters.getRunOrder(); + this.resolvedTests = resolvedTests; this.sortOrder = this.runOrder.length > 0 ? getSortOrderComparator( this.runOrder[0] ) : null; Long runOrderRandomSeed = runOrderParameters.getRunOrderRandomSeed(); if ( runOrderRandomSeed == null ) @@ -121,12 +127,25 @@ else if ( RunOrder.HOURLY.equals( runOrder ) ) final int hour = Calendar.getInstance().get( Calendar.HOUR_OF_DAY ); return ( ( hour % 2 ) == 0 ) ? getAlphabeticalComparator() : getReverseAlphabeticalComparator(); } + else if ( RunOrder.TEST.equals( runOrder ) ) + { + return getTestOrderComparator( resolvedTests ); + } else { return null; } } + private Comparator getTestOrderComparator( List resolvedTests ) + { + return Comparator.comparing( testClass -> IntStream.range( 0, resolvedTests.size() ) + .filter( i -> resolvedTests.get( i ).matchClass( testClass.getName().replace( '.', '/' ) ) ) + .findFirst() + .orElse( -1 ) + ); + } + private Comparator getReverseAlphabeticalComparator() { return new Comparator() diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrder.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrder.java index b0d938266a..5171c1d81c 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrder.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrder.java @@ -44,6 +44,8 @@ public class RunOrder public static final RunOrder FAILEDFIRST = new RunOrder( "failedfirst" ); + public static final RunOrder TEST = new RunOrder( "test" ); + public static final RunOrder[] DEFAULT = new RunOrder[]{ FILESYSTEM }; /** @@ -108,7 +110,8 @@ private static String createMessageForMissingRunOrder( String name ) private static RunOrder[] values() { - return new RunOrder[]{ ALPHABETICAL, FILESYSTEM, HOURLY, RANDOM, REVERSE_ALPHABETICAL, BALANCED, FAILEDFIRST }; + return new RunOrder[]{ ALPHABETICAL, FILESYSTEM, HOURLY, RANDOM, REVERSE_ALPHABETICAL, BALANCED, FAILEDFIRST, + TEST }; } public static String asString( RunOrder[] runOrder ) diff --git a/surefire-api/src/test/java/org/apache/maven/surefire/api/util/RunOrderCalculatorTest.java b/surefire-api/src/test/java/org/apache/maven/surefire/api/util/RunOrderCalculatorTest.java index 63768cf03d..00c9a440b1 100644 --- a/surefire-api/src/test/java/org/apache/maven/surefire/api/util/RunOrderCalculatorTest.java +++ b/surefire-api/src/test/java/org/apache/maven/surefire/api/util/RunOrderCalculatorTest.java @@ -19,13 +19,19 @@ * under the License. */ +import java.util.Collections; +import java.util.Iterator; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; +import org.apache.maven.surefire.api.testset.ResolvedTest; import org.apache.maven.surefire.api.testset.RunOrderParameters; import junit.framework.TestCase; +import static java.util.Arrays.asList; + /** * @author Kristian Rosenvold */ @@ -33,13 +39,36 @@ public class RunOrderCalculatorTest extends TestCase { - public void testOrderTestClasses() + public void testAlphabeticalOrder() { getClassesToRun(); TestsToRun testsToRun = new TestsToRun( getClassesToRun() ); - RunOrderCalculator runOrderCalculator = new DefaultRunOrderCalculator( RunOrderParameters.alphabetical(), 1 ); - final TestsToRun testsToRun1 = runOrderCalculator.orderTestClasses( testsToRun ); - assertEquals( A.class, testsToRun1.iterator().next() ); + RunOrderCalculator runOrderCalculator = new DefaultRunOrderCalculator( RunOrderParameters.alphabetical(), + 1, Collections.emptyList() ); + final TestsToRun orderedTestsToRun = runOrderCalculator.orderTestClasses( testsToRun ); + Iterator> iterator = orderedTestsToRun.iterator(); + assertEquals( A.class, iterator.next() ); + assertEquals( B.class, iterator.next() ); + assertEquals( C.class, iterator.next() ); + } + + public void testTestOrder() + { + TestsToRun testsToRun = new TestsToRun( getClassesToRun() ); + ResolvedTest testA = new ResolvedTest( A.class.getName(), null, false ); + ResolvedTest testB = new ResolvedTest( B.class.getName(), null, false ); + ResolvedTest testC = new ResolvedTest( C.class.getName(), null, false ); + + List resolvedTests = asList( testB, testC, testA ); + RunOrderParameters runOrderParameters = new RunOrderParameters( + RunOrder.valueOfMulti( "test" ), null ); + RunOrderCalculator runOrderCalculator = new DefaultRunOrderCalculator( runOrderParameters, + 1, resolvedTests ); + final TestsToRun orderedTestsToRun = runOrderCalculator.orderTestClasses( testsToRun ); + Iterator> iterator = orderedTestsToRun.iterator(); + assertEquals( B.class, iterator.next() ); + assertEquals( C.class, iterator.next() ); + assertEquals( A.class, iterator.next() ); } private Set> getClassesToRun() @@ -47,6 +76,7 @@ private Set> getClassesToRun() Set> classesToRun = new LinkedHashSet<>(); classesToRun.add( B.class ); classesToRun.add( A.class ); + classesToRun.add( C.class ); return classesToRun; } @@ -59,4 +89,9 @@ static class B { } + + static class C + { + + } }