Skip to content

Commit

Permalink
Fix JUnit5 reporting when tests are executed in parallel
Browse files Browse the repository at this point in the history
  • Loading branch information
ArvindJoshi-okta committed Aug 27, 2024
1 parent e99f11e commit 767e1a9
Show file tree
Hide file tree
Showing 9 changed files with 294 additions and 30 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.maven.surefire.its.jiras;

import org.apache.maven.surefire.its.fixture.OutputValidator;
import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
import org.junit.Test;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.is;

/**
* Integration tests for SUREFIRE-2252
*/
public class Surefire2252JUnit5IT extends SurefireJUnit4IntegrationTestCase {
@Test
public void testJUnit5() throws Exception {
unpack("surefire-2252-junit5-parallel")
.executeTest()
.verifyTextInLog(
"Using auto detected provider org.apache.maven.surefire.junitplatform.JUnitPlatformProvider")
.assertThatLogLine(containsString("Tests run: 1, Failures: 0, Errors: 0, Skipped: 0"), is(2));
}

@Test
public void testJUnit5Xml() {
OutputValidator validator =
unpack("surefire-2252-junit5-parallel-xml").executeTest().verifyErrorFree(2);

validator
.getSurefireReportsFile("TEST-pkg.domain.AxTest.xml", UTF_8)
.assertContainsText("tests=\"1\"")
.assertContainsText("errors=\"0\"")
.assertContainsText("skipped=\"0\"")
.assertContainsText("failures=\"0\"");

validator
.getSurefireReportsFile("TEST-pkg.domain.BxTest.xml", UTF_8)
.assertContainsText("tests=\"1\"")
.assertContainsText("errors=\"0\"")
.assertContainsText("skipped=\"0\"")
.assertContainsText("failures=\"0\"");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>org.example</groupId>
<artifactId>surefire-2252-junit5-parallel-xml</artifactId>
<version>1.0-SNAPSHOT</version>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>${java.specification.version}</maven.compiler.source>
<maven.compiler.target>${java.specification.version}</maven.compiler.target>
</properties>

<dependencies>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>5.10.2</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${surefire.version}</version>
<configuration>
<properties>
<configurationParameters>
junit.jupiter.execution.parallel.enabled = true
junit.jupiter.execution.parallel.mode.default = same_thread
junit.jupiter.execution.parallel.mode.classes.default = concurrent
</configurationParameters>
</properties>
<threadCount>2</threadCount>
<reportFormat>xml</reportFormat>
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>

</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package pkg.domain;

import org.junit.jupiter.api.Test;

class AxTest {
@Test
void test() {
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package pkg.domain;

import org.junit.jupiter.api.Test;

class BxTest {
@Test
void test() {
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>org.example</groupId>
<artifactId>surefire-2252-junit5-parallel</artifactId>
<version>1.0-SNAPSHOT</version>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>${java.specification.version}</maven.compiler.source>
<maven.compiler.target>${java.specification.version}</maven.compiler.target>
</properties>

<dependencies>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>5.10.2</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${surefire.version}</version>
<configuration>
<properties>
<configurationParameters>
junit.jupiter.execution.parallel.enabled = true
junit.jupiter.execution.parallel.mode.default = same_thread
junit.jupiter.execution.parallel.mode.classes.default = concurrent
</configurationParameters>
</properties>
<threadCount>2</threadCount>
<reportFormat>plain</reportFormat>
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>

</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package pkg.domain;

import org.junit.jupiter.api.Test;

class AxTest {
@Test
void test() {
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package pkg.domain;

import org.junit.jupiter.api.Test;

class BxTest {
@Test
void test() {
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
import org.apache.maven.surefire.api.util.SurefireReflectionException;
import org.apache.maven.surefire.api.util.TestsToRun;
import org.apache.maven.surefire.shared.utils.StringUtils;
import org.junit.platform.engine.DiscoverySelector;
import org.junit.platform.engine.Filter;
import org.junit.platform.launcher.EngineFilter;
import org.junit.platform.launcher.Launcher;
Expand Down Expand Up @@ -172,25 +171,13 @@ private void invokeAllTests(TestsToRun testsToRun, RunListenerAdapter adapter) {
}

private void execute(TestsToRun testsToRun, RunListenerAdapter adapter) {
if (testsToRun.allowEagerReading()) {
List<DiscoverySelector> selectors = new ArrayList<>();
testsToRun.iterator().forEachRemaining(c -> selectors.add(selectClass(c.getName())));

testsToRun.iterator().forEachRemaining(c -> {
LauncherDiscoveryRequestBuilder builder = request()
.filters(filters)
.configurationParameters(configurationParameters)
.selectors(selectors);

.selectors(selectClass(c.getName()));
launcher.execute(builder.build(), adapter);
} else {
testsToRun.iterator().forEachRemaining(c -> {
LauncherDiscoveryRequestBuilder builder = request()
.filters(filters)
.configurationParameters(configurationParameters)
.selectors(selectClass(c.getName()));
launcher.execute(builder.build(), adapter);
});
}
});
}

private void closeLauncher() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -199,14 +199,22 @@ public void allGivenTestsToRunAreInvoked() throws Exception {
TestsToRun testsToRun = newTestsToRun(TestClass1.class, TestClass2.class);
invokeProvider(provider, testsToRun);

assertThat(executionListener.summaries).hasSize(1);
assertThat(executionListener.summaries).hasSize(2);
TestExecutionSummary summary = executionListener.summaries.get(0);
assertEquals(TestClass1.TESTS_FOUND + TestClass2.TESTS_FOUND, summary.getTestsFoundCount());
assertEquals(TestClass1.TESTS_STARTED + TestClass2.TESTS_STARTED, summary.getTestsStartedCount());
assertEquals(TestClass1.TESTS_SKIPPED + TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount());
assertEquals(TestClass1.TESTS_SUCCEEDED + TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
assertEquals(TestClass1.TESTS_ABORTED + TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount());
assertEquals(TestClass1.TESTS_FAILED + TestClass2.TESTS_FAILED, summary.getTestsFailedCount());
assertEquals(TestClass1.TESTS_FOUND, summary.getTestsFoundCount());
assertEquals(TestClass1.TESTS_STARTED, summary.getTestsStartedCount());
assertEquals(TestClass1.TESTS_SKIPPED, summary.getTestsSkippedCount());
assertEquals(TestClass1.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
assertEquals(TestClass1.TESTS_ABORTED, summary.getTestsAbortedCount());
assertEquals(TestClass1.TESTS_FAILED, summary.getTestsFailedCount());

summary = executionListener.summaries.get(1);
assertEquals(TestClass2.TESTS_FOUND, summary.getTestsFoundCount());
assertEquals(TestClass2.TESTS_STARTED, summary.getTestsStartedCount());
assertEquals(TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount());
assertEquals(TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
assertEquals(TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount());
assertEquals(TestClass2.TESTS_FAILED, summary.getTestsFailedCount());
}

@Test
Expand Down Expand Up @@ -249,6 +257,34 @@ public void singleTestClassIsInvokedLazily() throws Exception {
assertEquals(TestClass1.TESTS_FAILED, summary.getTestsFailedCount());
}

@Test
public void multipleTestClassesAreInvokedLazily() throws Exception {
Launcher launcher = LauncherFactory.create();
JUnitPlatformProvider provider = new JUnitPlatformProvider(providerParametersMock(), launcher);

TestPlanSummaryListener executionListener = new TestPlanSummaryListener();
launcher.registerTestExecutionListeners(executionListener);

invokeProvider(provider, newTestsToRunLazily(TestClass1.class, TestClass2.class));

assertThat(executionListener.summaries).hasSize(2);
TestExecutionSummary summary = executionListener.summaries.get(0);
assertEquals(TestClass1.TESTS_FOUND, summary.getTestsFoundCount());
assertEquals(TestClass1.TESTS_STARTED, summary.getTestsStartedCount());
assertEquals(TestClass1.TESTS_SKIPPED, summary.getTestsSkippedCount());
assertEquals(TestClass1.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
assertEquals(TestClass1.TESTS_ABORTED, summary.getTestsAbortedCount());
assertEquals(TestClass1.TESTS_FAILED, summary.getTestsFailedCount());

summary = executionListener.summaries.get(1);
assertEquals(TestClass2.TESTS_FOUND, summary.getTestsFoundCount());
assertEquals(TestClass2.TESTS_STARTED, summary.getTestsStartedCount());
assertEquals(TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount());
assertEquals(TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
assertEquals(TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount());
assertEquals(TestClass2.TESTS_FAILED, summary.getTestsFailedCount());
}

@Test
public void failingTestCaseAfterRerun() throws Exception {
Launcher launcher = LauncherFactory.create();
Expand Down Expand Up @@ -682,14 +718,22 @@ public void allDiscoveredTestsAreInvokedForNullArgument() throws Exception {
assertThat(report.getValue().getSourceName()).isEqualTo(TestClass2.class.getName());
assertThat(report.getValue().getName()).isNull();

assertThat(executionListener.summaries).hasSize(1);
assertThat(executionListener.summaries).hasSize(2);
TestExecutionSummary summary = executionListener.summaries.get(0);
assertEquals(TestClass1.TESTS_FOUND + TestClass2.TESTS_FOUND, summary.getTestsFoundCount());
assertEquals(TestClass1.TESTS_STARTED + TestClass2.TESTS_STARTED, summary.getTestsStartedCount());
assertEquals(TestClass1.TESTS_SKIPPED + TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount());
assertEquals(TestClass1.TESTS_SUCCEEDED + TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
assertEquals(TestClass1.TESTS_ABORTED + TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount());
assertEquals(TestClass1.TESTS_FAILED + TestClass2.TESTS_FAILED, summary.getTestsFailedCount());
assertEquals(TestClass1.TESTS_FOUND, summary.getTestsFoundCount());
assertEquals(TestClass1.TESTS_STARTED, summary.getTestsStartedCount());
assertEquals(TestClass1.TESTS_SKIPPED, summary.getTestsSkippedCount());
assertEquals(TestClass1.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
assertEquals(TestClass1.TESTS_ABORTED, summary.getTestsAbortedCount());
assertEquals(TestClass1.TESTS_FAILED, summary.getTestsFailedCount());

summary = executionListener.summaries.get(1);
assertEquals(TestClass2.TESTS_FOUND, summary.getTestsFoundCount());
assertEquals(TestClass2.TESTS_STARTED, summary.getTestsStartedCount());
assertEquals(TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount());
assertEquals(TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
assertEquals(TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount());
assertEquals(TestClass2.TESTS_FAILED, summary.getTestsFailedCount());
}

@Test
Expand Down

0 comments on commit 767e1a9

Please sign in to comment.