Fix JUnit5 reporting when tests are executed in parallel

apache · Aug 23, 2024 · 5c627f9 · 5c627f9
1 parent e99f11e
commit 5c627f9
Show file tree

Hide file tree

Showing 9 changed files with 300 additions and 21 deletions.
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2252JUnit5IT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2252JUnit5IT.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.maven.surefire.its.jiras;
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Integration tests for SUREFIRE-2252
+ */
+public class Surefire2252JUnit5IT extends SurefireJUnit4IntegrationTestCase {
+    @Test
+    public void testJUnit5() throws Exception {
+        unpack("surefire-2252-junit5-parallel")
+                .executeTest()
+                .verifyTextInLog(
+                        "Using auto detected provider org.apache.maven.surefire.junitplatform.JUnitPlatformProvider")
+                .assertThatLogLine(containsString("Tests run: 1, Failures: 0, Errors: 0, Skipped: 0"), is(2));
+    }
+
+    @Test
+    public void testJUnit5Xml() {
+        OutputValidator validator =
+                unpack("surefire-2252-junit5-parallel-xml").executeTest().verifyErrorFree(2);
+
+        validator
+                .getSurefireReportsFile("TEST-pkg.domain.AxTest.xml", UTF_8)
+                .assertContainsText("tests=\"1\"")
+                .assertContainsText("errors=\"0\"")
+                .assertContainsText("skipped=\"0\"")
+                .assertContainsText("failures=\"0\"");
+
+        validator
+                .getSurefireReportsFile("TEST-pkg.domain.BxTest.xml", UTF_8)
+                .assertContainsText("tests=\"1\"")
+                .assertContainsText("errors=\"0\"")
+                .assertContainsText("skipped=\"0\"")
+                .assertContainsText("failures=\"0\"");
+    }
+}
diff --git a/surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/pom.xml b/surefire-its/src/test/resources/surefire-2252-junit5-parallel-xml/pom.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.example</groupId>
+  <artifactId>surefire-2252-junit5-parallel</artifactId>
+  <version>1.0-SNAPSHOT</version>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <maven.compiler.source>${java.specification.version}</maven.compiler.source>
+    <maven.compiler.target>${java.specification.version}</maven.compiler.target>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <version>5.10.2</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-surefire-plugin</artifactId>
+          <version>3.3.2-SNAPSHOT</version>
+          <configuration>
+            <properties>
+              <configurationParameters>
+                junit.jupiter.execution.parallel.enabled = true
+                junit.jupiter.execution.parallel.mode.default = same_thread
+                junit.jupiter.execution.parallel.mode.classes.default = concurrent
+              </configurationParameters>
+            </properties>
+            <threadCount>2</threadCount>
+            <reportFormat>xml</reportFormat>
+          </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+
+</project>
diff --git a/...src/test/resources/surefire-2252-junit5-parallel-xml/src/test/java/pkg/domain/AxTest.java b/...src/test/resources/surefire-2252-junit5-parallel-xml/src/test/java/pkg/domain/AxTest.java
@@ -0,0 +1,9 @@
+package pkg.domain;
+
+import org.junit.jupiter.api.Test;
+
+class AxTest {
+    @Test
+    void test() {
+    }
+}
diff --git a/...src/test/resources/surefire-2252-junit5-parallel-xml/src/test/java/pkg/domain/BxTest.java b/...src/test/resources/surefire-2252-junit5-parallel-xml/src/test/java/pkg/domain/BxTest.java
@@ -0,0 +1,9 @@
+package pkg.domain;
+
+import org.junit.jupiter.api.Test;
+
+class BxTest {
+    @Test
+    void test() {
+    }
+}
diff --git a/surefire-its/src/test/resources/surefire-2252-junit5-parallel/pom.xml b/surefire-its/src/test/resources/surefire-2252-junit5-parallel/pom.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.example</groupId>
+  <artifactId>surefire-2252-junit5-parallel</artifactId>
+  <version>1.0-SNAPSHOT</version>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <maven.compiler.source>${java.specification.version}</maven.compiler.source>
+    <maven.compiler.target>${java.specification.version}</maven.compiler.target>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <version>5.10.2</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-surefire-plugin</artifactId>
+          <version>3.3.2-SNAPSHOT</version>
+          <configuration>
+            <properties>
+              <configurationParameters>
+                junit.jupiter.execution.parallel.enabled = true
+                junit.jupiter.execution.parallel.mode.default = same_thread
+                junit.jupiter.execution.parallel.mode.classes.default = concurrent
+              </configurationParameters>
+            </properties>
+            <threadCount>2</threadCount>
+            <reportFormat>plain</reportFormat>
+          </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+
+</project>
diff --git a/...its/src/test/resources/surefire-2252-junit5-parallel/src/test/java/pkg/domain/AxTest.java b/...its/src/test/resources/surefire-2252-junit5-parallel/src/test/java/pkg/domain/AxTest.java
@@ -0,0 +1,9 @@
+package pkg.domain;
+
+import org.junit.jupiter.api.Test;
+
+class AxTest {
+    @Test
+    void test() {
+    }
+}
diff --git a/...its/src/test/resources/surefire-2252-junit5-parallel/src/test/java/pkg/domain/BxTest.java b/...its/src/test/resources/surefire-2252-junit5-parallel/src/test/java/pkg/domain/BxTest.java
@@ -0,0 +1,9 @@
+package pkg.domain;
+
+import org.junit.jupiter.api.Test;
+
+class BxTest {
+    @Test
+    void test() {
+    }
+}
diff --git a/...platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java b/...platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java
@@ -173,15 +173,17 @@ private void invokeAllTests(TestsToRun testsToRun, RunListenerAdapter adapter) {
 
     private void execute(TestsToRun testsToRun, RunListenerAdapter adapter) {
         if (testsToRun.allowEagerReading()) {
-            List<DiscoverySelector> selectors = new ArrayList<>();
-            testsToRun.iterator().forEachRemaining(c -> selectors.add(selectClass(c.getName())));
+            testsToRun.iterator().forEachRemaining(c -> {
+                List<DiscoverySelector> selectors = new ArrayList<>();
+                selectors.add(selectClass(c.getName()));
 
-            LauncherDiscoveryRequestBuilder builder = request()
-                    .filters(filters)
-                    .configurationParameters(configurationParameters)
-                    .selectors(selectors);
+                LauncherDiscoveryRequestBuilder builder = request()
+                        .filters(filters)
+                        .configurationParameters(configurationParameters)
+                        .selectors(selectors);
 
-            launcher.execute(builder.build(), adapter);
+                launcher.execute(builder.build(), adapter);
+            });
         } else {
             testsToRun.iterator().forEachRemaining(c -> {
                 LauncherDiscoveryRequestBuilder builder = request()

diff --git a/...form/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java b/...form/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java
@@ -199,14 +199,22 @@ public void allGivenTestsToRunAreInvoked() throws Exception {
         TestsToRun testsToRun = newTestsToRun(TestClass1.class, TestClass2.class);
         invokeProvider(provider, testsToRun);
 
-        assertThat(executionListener.summaries).hasSize(1);
+        assertThat(executionListener.summaries).hasSize(2);
         TestExecutionSummary summary = executionListener.summaries.get(0);
-        assertEquals(TestClass1.TESTS_FOUND + TestClass2.TESTS_FOUND, summary.getTestsFoundCount());
-        assertEquals(TestClass1.TESTS_STARTED + TestClass2.TESTS_STARTED, summary.getTestsStartedCount());
-        assertEquals(TestClass1.TESTS_SKIPPED + TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount());
-        assertEquals(TestClass1.TESTS_SUCCEEDED + TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
-        assertEquals(TestClass1.TESTS_ABORTED + TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount());
-        assertEquals(TestClass1.TESTS_FAILED + TestClass2.TESTS_FAILED, summary.getTestsFailedCount());
+        assertEquals(TestClass1.TESTS_FOUND, summary.getTestsFoundCount());
+        assertEquals(TestClass1.TESTS_STARTED, summary.getTestsStartedCount());
+        assertEquals(TestClass1.TESTS_SKIPPED, summary.getTestsSkippedCount());
+        assertEquals(TestClass1.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
+        assertEquals(TestClass1.TESTS_ABORTED, summary.getTestsAbortedCount());
+        assertEquals(TestClass1.TESTS_FAILED, summary.getTestsFailedCount());
+
+        summary = executionListener.summaries.get(1);
+        assertEquals(TestClass2.TESTS_FOUND, summary.getTestsFoundCount());
+        assertEquals(TestClass2.TESTS_STARTED, summary.getTestsStartedCount());
+        assertEquals(TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount());
+        assertEquals(TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
+        assertEquals(TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount());
+        assertEquals(TestClass2.TESTS_FAILED, summary.getTestsFailedCount());
     }
 
     @Test
@@ -249,6 +257,34 @@ public void singleTestClassIsInvokedLazily() throws Exception {
         assertEquals(TestClass1.TESTS_FAILED, summary.getTestsFailedCount());
     }
 
+    @Test
+    public void multipleTestClassesIsInvokedLazily() throws Exception {
+        Launcher launcher = LauncherFactory.create();
+        JUnitPlatformProvider provider = new JUnitPlatformProvider(providerParametersMock(), launcher);
+
+        TestPlanSummaryListener executionListener = new TestPlanSummaryListener();
+        launcher.registerTestExecutionListeners(executionListener);
+
+        invokeProvider(provider, newTestsToRunLazily(TestClass1.class, TestClass2.class));
+
+        assertThat(executionListener.summaries).hasSize(2);
+        TestExecutionSummary summary = executionListener.summaries.get(0);
+        assertEquals(TestClass1.TESTS_FOUND, summary.getTestsFoundCount());
+        assertEquals(TestClass1.TESTS_STARTED, summary.getTestsStartedCount());
+        assertEquals(TestClass1.TESTS_SKIPPED, summary.getTestsSkippedCount());
+        assertEquals(TestClass1.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
+        assertEquals(TestClass1.TESTS_ABORTED, summary.getTestsAbortedCount());
+        assertEquals(TestClass1.TESTS_FAILED, summary.getTestsFailedCount());
+
+        summary = executionListener.summaries.get(1);
+        assertEquals(TestClass2.TESTS_FOUND, summary.getTestsFoundCount());
+        assertEquals(TestClass2.TESTS_STARTED, summary.getTestsStartedCount());
+        assertEquals(TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount());
+        assertEquals(TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
+        assertEquals(TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount());
+        assertEquals(TestClass2.TESTS_FAILED, summary.getTestsFailedCount());
+    }
+
     @Test
     public void failingTestCaseAfterRerun() throws Exception {
         Launcher launcher = LauncherFactory.create();
@@ -682,14 +718,22 @@ public void allDiscoveredTestsAreInvokedForNullArgument() throws Exception {
         assertThat(report.getValue().getSourceName()).isEqualTo(TestClass2.class.getName());
         assertThat(report.getValue().getName()).isNull();
 
-        assertThat(executionListener.summaries).hasSize(1);
+        assertThat(executionListener.summaries).hasSize(2);
         TestExecutionSummary summary = executionListener.summaries.get(0);
-        assertEquals(TestClass1.TESTS_FOUND + TestClass2.TESTS_FOUND, summary.getTestsFoundCount());
-        assertEquals(TestClass1.TESTS_STARTED + TestClass2.TESTS_STARTED, summary.getTestsStartedCount());
-        assertEquals(TestClass1.TESTS_SKIPPED + TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount());
-        assertEquals(TestClass1.TESTS_SUCCEEDED + TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
-        assertEquals(TestClass1.TESTS_ABORTED + TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount());
-        assertEquals(TestClass1.TESTS_FAILED + TestClass2.TESTS_FAILED, summary.getTestsFailedCount());
+        assertEquals(TestClass1.TESTS_FOUND, summary.getTestsFoundCount());
+        assertEquals(TestClass1.TESTS_STARTED, summary.getTestsStartedCount());
+        assertEquals(TestClass1.TESTS_SKIPPED, summary.getTestsSkippedCount());
+        assertEquals(TestClass1.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
+        assertEquals(TestClass1.TESTS_ABORTED, summary.getTestsAbortedCount());
+        assertEquals(TestClass1.TESTS_FAILED, summary.getTestsFailedCount());
+
+        summary = executionListener.summaries.get(1);
+        assertEquals(TestClass2.TESTS_FOUND, summary.getTestsFoundCount());
+        assertEquals(TestClass2.TESTS_STARTED, summary.getTestsStartedCount());
+        assertEquals(TestClass2.TESTS_SKIPPED, summary.getTestsSkippedCount());
+        assertEquals(TestClass2.TESTS_SUCCEEDED, summary.getTestsSucceededCount());
+        assertEquals(TestClass2.TESTS_ABORTED, summary.getTestsAbortedCount());
+        assertEquals(TestClass2.TESTS_FAILED, summary.getTestsFailedCount());
     }
 
     @Test