Sort input files in the test runner.

While the order of the input files _shouldn't_ affect behavior, it sometimes does. Let's use a consistent order so that we at least don't see behavior differences between local and CI runs: #165 (comment)
jspecify · Apr 2, 2024 · 9945f0f · 9945f0f
1 parent 75aa1de
commit 9945f0f
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/conformance-test-framework/src/main/java/org/jspecify/conformance/ConformanceTestRunner.java b/conformance-test-framework/src/main/java/org/jspecify/conformance/ConformanceTestRunner.java
@@ -27,6 +27,7 @@
 
 import com.google.common.base.Ascii;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSortedSet;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
@@ -48,7 +49,7 @@ public interface Analyzer {
      * @return the facts reported by the analysis
      */
     Iterable<ReportedFact> analyze(
-        Path testDirectory, ImmutableList<Path> files, ImmutableList<Path> testDeps);
+        Path testDirectory, ImmutableSortedSet<Path> files, ImmutableList<Path> testDeps);
   }
 
   private final Analyzer analyzer;
@@ -77,7 +78,7 @@ public ConformanceTestReport runTests(Path testDirectory, ImmutableList<Path> te
                     ? groups.flatMap(files -> partition(files, 1).stream())
                     : groups;
               })
-          .map(ImmutableList::copyOf)
+          .map(ImmutableSortedSet::copyOf)
           .forEach(
               files -> report.addFiles(files, analyzer.analyze(testDirectory, files, testDeps)));
       return report.build();

diff --git a/src/test/java/tests/ConformanceTest.java b/src/test/java/tests/ConformanceTest.java
@@ -24,6 +24,7 @@
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
 import com.google.jspecify.nullness.NullSpecChecker;
 import java.io.IOException;
 import java.nio.file.Path;
@@ -130,7 +131,7 @@ private static Path systemPropertyPath(String key, String description) {
   }
 
   private static ImmutableSet<ReportedFact> analyze(
-      Path testDirectory, ImmutableList<Path> files, ImmutableList<Path> testDeps) {
+      Path testDirectory, ImmutableSortedSet<Path> files, ImmutableList<Path> testDeps) {
     TestConfiguration config =
         TestConfigurationBuilder.buildDefaultConfiguration(
             null,