diff --git a/changelog/@unreleased/pr-2391.v2.yml b/changelog/@unreleased/pr-2391.v2.yml
new file mode 100644
index 000000000..73d47a513
--- /dev/null
+++ b/changelog/@unreleased/pr-2391.v2.yml
@@ -0,0 +1,6 @@
+type: fix
+fix:
+  description: Restore the JUnit reports plugin by default so checkstyle, compile
+    errors and gradle task failures appear in an easy to read format on Circle.
+  links:
+  - https://github.com/palantir/gradle-baseline/pull/2391
diff --git a/gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/BaselineCircleCi.java b/gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/BaselineCircleCi.java
index 2c496732d..9d500e39d 100644
--- a/gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/BaselineCircleCi.java
+++ b/gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/BaselineCircleCi.java
@@ -19,6 +19,8 @@
 import com.google.common.base.Preconditions;
 import com.google.common.base.Splitter;
 import com.palantir.gradle.junit.JunitReportsExtension;
+import com.palantir.gradle.junit.JunitReportsPlugin;
+import com.palantir.gradle.junit.JunitReportsRootPlugin;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
@@ -38,6 +40,9 @@ public final class BaselineCircleCi implements Plugin<Project> {
 
     @Override
     public void apply(Project project) {
+        project.getPluginManager().apply(JunitReportsRootPlugin.class);
+        project.getPluginManager().apply(JunitReportsPlugin.class);
+
         configurePluginsForReports(project);
         configurePluginsForArtifacts(project);
 
@@ -79,20 +84,9 @@ private void configurePluginsForReports(Project project) {
             throw new RuntimeException("failed to create CIRCLE_TEST_REPORTS directory", e);
         }
 
-        project.getRootProject()
-                .allprojects(proj -> proj.getTasks().withType(Test.class).configureEach(test -> {
-                    test.getReports().getJunitXml().getRequired().set(true);
-                    test.getReports()
-                            .getJunitXml()
-                            .getOutputLocation()
-                            .set(junitPath(circleReportsDir, test.getPath()));
-                }));
-
-        project.getPluginManager().withPlugin("com.palantir.junit-reports", unused -> {
-            project.getExtensions().configure(JunitReportsExtension.class, junitReports -> junitReports
-                    .getReportsDirectory()
-                    .set(new File(circleReportsDir)));
-        });
+        project.getExtensions().configure(JunitReportsExtension.class, junitReports -> junitReports
+                .getReportsDirectory()
+                .set(new File(circleReportsDir)));
     }
 
     private static File junitPath(String basePath, String testPath) {
diff --git a/gradle-baseline-java/src/test/groovy/com/palantir/baseline/BaselineCircleCiJavaIntegrationTests.java b/gradle-baseline-java/src/test/groovy/com/palantir/baseline/BaselineCircleCiJavaIntegrationTests.java
index db690f007..128b90433 100644
--- a/gradle-baseline-java/src/test/groovy/com/palantir/baseline/BaselineCircleCiJavaIntegrationTests.java
+++ b/gradle-baseline-java/src/test/groovy/com/palantir/baseline/BaselineCircleCiJavaIntegrationTests.java
@@ -53,6 +53,34 @@ public void before() {
         copyTestFile("checkstyle.xml", projectDir, "config/checkstyle/checkstyle.xml");
     }
 
+    @Test
+    public void javacIntegrationTest() throws IOException {
+        copyTestFile("non-compiling-class", projectDir, "src/main/java/com/example/MyClass.java");
+
+        BuildResult result = GradleRunner.create()
+                .withPluginClasspath()
+                .withProjectDir(projectDir.getRoot())
+                .withArguments("--stacktrace", "compileJava")
+                .buildAndFail();
+        assertThat(result.getOutput())
+                .contains("Compilation failed")
+                .contains("error: incompatible types")
+                .contains("private final int a")
+                .contains("error: cannot assign a value to final variable b")
+                .contains("b = 2")
+                .contains("uses unchecked or unsafe operations");
+
+        File report = new File(reportsDir, "/foobar-compileJava.xml");
+        assertThat(report).exists();
+        String reportXml = Files.asCharSource(report, StandardCharsets.UTF_8).read();
+        assertThat(reportXml)
+                .contains("incompatible types")
+                .contains("private final int a")
+                .contains("cannot assign a value to final variable b")
+                .contains("b = 2")
+                .doesNotContain("uses unchecked or unsafe operations");
+    }
+
     @Test
     public void junitIntegrationTest() throws IOException {
         copyTestFile("tested-class", projectDir, "src/main/java/com/example/MyClass.java");
@@ -95,6 +123,57 @@ public void junitSubprojectIntegrationTest() throws IOException {
                 .contains("org.junit.ComparisonFailure");
     }
 
+    @Test
+    public void checkstyleIntegrationTest() throws IOException {
+        copyTestFile("checkstyle-violating-class", projectDir, "src/main/java/com/example/MyClass.java");
+
+        BuildResult result = GradleRunner.create()
+                .withPluginClasspath()
+                .withProjectDir(projectDir.getRoot())
+                .withArguments("--stacktrace", "checkstyleMain")
+                .buildAndFail();
+        assertThat(result.getOutput()).contains("Checkstyle rule violations were found");
+
+        File report = new File(reportsDir, "foobar-checkstyleMain.xml");
+        assertThat(report).exists();
+        String reportXml = Files.asCharSource(report, StandardCharsets.UTF_8).read();
+        assertThat(reportXml).contains("Name 'a_constant' must match pattern");
+    }
+
+    @Test
+    public void buildStepFailureIntegrationTest() throws IOException {
+        BuildResult result = GradleRunner.create()
+                .withPluginClasspath()
+                .withProjectDir(projectDir.getRoot())
+                .withArguments("--stacktrace", "failingTask")
+                .buildAndFail();
+        assertThat(result.getOutput()).contains("This task will always fail");
+
+        File report = new File(reportsDir, "gradle/build.xml");
+        assertThat(report).exists();
+        String reportXml = Files.asCharSource(report, StandardCharsets.UTF_8).read();
+        assertThat(reportXml).contains("message=\"RuntimeException: This task will always fail\"");
+    }
+
+    @Test
+    public void findsUniqueBuildStepsReportFileName() throws IOException {
+        assertThat(new File(reportsDir, "gradle").mkdirs()).isTrue();
+        assertThat(new File(reportsDir, "gradle/build.xml").createNewFile()).isTrue();
+        assertThat(new File(reportsDir, "gradle/build2.xml").createNewFile()).isTrue();
+
+        BuildResult result = GradleRunner.create()
+                .withPluginClasspath()
+                .withProjectDir(projectDir.getRoot())
+                .withArguments("--stacktrace", "failingTask")
+                .buildAndFail();
+        assertThat(result.getOutput()).contains("This task will always fail");
+
+        File report = new File(reportsDir, "gradle/build3.xml");
+        assertThat(report).exists();
+        String reportXml = Files.asCharSource(report, StandardCharsets.UTF_8).read();
+        assertThat(reportXml).contains("message=\"RuntimeException: This task will always fail\"");
+    }
+
     @Test
     public void canCallGradleThreeTimesInARow() {
         GradleRunner.create()
diff --git a/gradle-junit-reports/src/main/resources/META-INF/gradle-plugins/com.palantir.junit-reports.properties b/gradle-junit-reports/src/main/resources/META-INF/gradle-plugins/com.palantir.junit-reports.properties
index b94b2b360..4bb115cdb 100644
--- a/gradle-junit-reports/src/main/resources/META-INF/gradle-plugins/com.palantir.junit-reports.properties
+++ b/gradle-junit-reports/src/main/resources/META-INF/gradle-plugins/com.palantir.junit-reports.properties
@@ -1 +1 @@
-implementation-class=com.palantir.gradle.junit.JunitReportsRootPlugin
+implementation-class=com.palantir.gradle.junit.JunitReportsPlugin