Add generate-time suite sharding (--shard-count / --shard-index)

README.md

@@ -187,6 +187,20 @@ Exclusion maps to setting `[Ignore]` attribute in test suite.
 
 Non-parallel marking maps to setting `[NonParallelizable]` on the affected generated test method(s). Because all tests under a given sub-directory (e.g. `built-ins/Atomics/waitAsync`) collapse into one generated method, marking any one entry serialises the whole group — sufficient for timing-sensitive feature suites such as `Atomics.waitAsync`. File entries use the test262 forward-slash path format; esprima-style `(default)` / `(strict mode)` suffixes are not supported here (the attribute is method-level).
 
+## Sharding the generated suite
+
+`generate` accepts two command-line options for splitting the suite into independent slices so a CI matrix can compile and run them on separate runners in parallel:
+
+| Option           | Default | Description                                                     |
+|:-----------------|:--------|:----------------------------------------------------------------|
+| `--shard-count`  | 1       | Total number of shards to split the suite into (1 = no sharding) |
+| `--shard-index`  | 0       | Zero-based index of the shard to emit, in range `[0, shard-count)` |
+
+```
+dotnet test262 generate --shard-count 4 --shard-index 0
+```
+
+Each test file is assigned to a shard by a stable, order-independent hash of its name, so the assignment is deterministic across runs, machines and operating systems: shard *k* always contains the same files, the shards never overlap, and their union is exactly the full (unsharded) suite. Both strict and non-strict variants of a file always land in the same shard. Because the assignment does not depend on where generation runs, a shard generated on one operating system can be reused (e.g. from a cross-OS cache) by test jobs on any other.
 
 ## Branches and releases
 

src/Test262Harness.Console/Program.cs

@@ -51,6 +51,14 @@ public sealed class Settings : CommandSettings
         [CommandOption("-s|--settings")]
         [DefaultValue("Test262Harness.settings.json")]
         public string SettingsFile { get; set; } = "";
+
+        [Description("Total number of shards to split the suite into (default 1 = no sharding)")]
+        [CommandOption("--shard-count")]
+        public int? ShardCount { get; set; }
+
+        [Description("Zero-based index of the shard to emit, in range [0, shard-count)")]
+        [CommandOption("--shard-index")]
+        public int? ShardIndex { get; set; }
     }
 
     protected override async Task<int> ExecuteAsync([NotNull] CommandContext context, [NotNull] Settings settings, CancellationToken cancellationToken)
@@ -152,6 +160,16 @@ private static async Task FinalizeOptions(Settings settings, TestSuiteGeneratorO
             options.ExcludedFilesSource = settings.ExcludedFilesSource;
         }
 
+        if (settings.ShardCount is not null)
+        {
+            options.ShardCount = settings.ShardCount.Value;
+        }
+
+        if (settings.ShardIndex is not null)
+        {
+            options.ShardIndex = settings.ShardIndex.Value;
+        }
+
         if (!string.IsNullOrWhiteSpace(options.ExcludedFilesSource))
         {
             var lines = await File.ReadAllLinesAsync(options.ExcludedFilesSource);

src/Test262Harness.Generator/TestSuiteGenerator.cs

@@ -32,6 +32,16 @@ public class TestSuiteGenerator
 
     public TestSuiteGenerator(TestSuiteGeneratorOptions options, string? usedSettingsFilePath)
     {
+        if (options.ShardCount < 1)
+        {
+            throw new ArgumentException($"ShardCount must be >= 1 but was {options.ShardCount}.", nameof(options));
+        }
+
+        if (options.ShardIndex < 0 || options.ShardIndex >= options.ShardCount)
+        {
+            throw new ArgumentException($"ShardIndex must be in the range [0, {options.ShardCount}) but was {options.ShardIndex}.", nameof(options));
+        }
+
         _options = options;
         _usedSettingsFilePath = usedSettingsFilePath;
 
@@ -116,6 +126,11 @@ [.. _options.NonParallelFiles
         {
             var tests = stream.GetTestFiles([item]).ToList();
 
+            if (_options.ShardCount > 1)
+            {
+                tests = tests.Where(x => GetShardIndex(x.FileName) == _options.ShardIndex).ToList();
+            }
+
             if (tests.Count == 0)
             {
                 continue;
@@ -205,6 +220,32 @@ private void SetCommonInfo(TemplateContext context, string templateHash)
         context.SetValue("TemplateSha", templateHash);
     }
 
+    private int GetShardIndex(string fileName)
+    {
+        return (int) (StableHash(fileName) % (uint) _options.ShardCount);
+    }
+
+    /// <summary>
+    /// A small, deterministic hash (FNV-1a, 32-bit) used to assign a test file to a shard.
+    /// Unlike <see cref="string.GetHashCode()"/> — which is randomized per process on modern .NET —
+    /// this produces identical results across runs, machines and operating systems, so a given file
+    /// always lands in the same shard regardless of where generation happens.
+    /// </summary>
+    private static uint StableHash(string value)
+    {
+        const uint offsetBasis = 2166136261;
+        const uint prime = 16777619;
+
+        var hash = offsetBasis;
+        foreach (var c in value)
+        {
+            hash = (hash ^ (byte) c) * prime;
+            hash = (hash ^ (byte) (c >> 8)) * prime;
+        }
+
+        return hash;
+    }
+
     private static string StripTestPrefix(string fileName)
     {
         return fileName.StartsWith(TestPrefix, StringComparison.OrdinalIgnoreCase)

src/Test262Harness.Generator/TestSuiteGeneratorOptions.cs

@@ -25,4 +25,21 @@ public class TestSuiteGeneratorOptions
     public string[] NonParallelFiles { get; set; } = [];
 
     public string ExcludedFilesSource { get; set; } = "";
+
+    /// <summary>
+    /// Total number of shards to split the generated suite into. Defaults to 1 (no sharding).
+    /// When greater than 1, each generation only emits the test cases belonging to <see cref="ShardIndex"/>,
+    /// letting a CI matrix compile and run distinct slices of the suite in parallel. The assignment is a
+    /// stable, order-independent hash of the test file name, so the shards are deterministic across runs,
+    /// machines and operating systems, and their union is exactly the full (unsharded) suite.
+    /// </summary>
+    [JsonIgnore]
+    public int ShardCount { get; set; } = 1;
+
+    /// <summary>
+    /// Zero-based index of the shard to emit when <see cref="ShardCount"/> is greater than 1.
+    /// Must be in the range <c>[0, ShardCount)</c>.
+    /// </summary>
+    [JsonIgnore]
+    public int ShardIndex { get; set; }
 }

test/Test262Harness.Tests/TestSuiteGeneratorTests.cs

@@ -1,4 +1,5 @@
 using System.Runtime.CompilerServices;
+using System.Text.RegularExpressions;
 using Test262Harness.TestSuite.Generator;
 using Zio;
 using Zio.FileSystems;
@@ -140,4 +141,128 @@ public async Task IgnoredGroupDoesNotGetNonParallelizable()
         });
         await VerifyOutput(output);
     }
+
+    [Test]
+    public async Task ShardingPartitionsSuiteWithoutOverlapOrLoss()
+    {
+        const int fileCount = 40;
+        const int shardCount = 4;
+
+        var all = await GenerateShardedTestFileNames(shardCount: 1, shardIndex: 0, fileCount);
+        Assert.That(all, Has.Count.EqualTo(fileCount));
+
+        var union = new HashSet<string>(StringComparer.Ordinal);
+        var perShardCounts = new List<int>();
+        for (var shardIndex = 0; shardIndex < shardCount; shardIndex++)
+        {
+            var shard = await GenerateShardedTestFileNames(shardCount, shardIndex, fileCount);
+            perShardCounts.Add(shard.Count);
+            foreach (var fileName in shard)
+            {
+                Assert.That(union.Add(fileName), Is.True, $"File {fileName} appeared in more than one shard");
+            }
+        }
+
+        Assert.That(union, Is.EquivalentTo(all), "Union of all shards must equal the full (unsharded) suite");
+        Assert.That(perShardCounts, Has.All.GreaterThan(0), "Every shard should receive at least one test");
+    }
+
+    [Test]
+    public async Task ShardAssignmentIsDeterministic()
+    {
+        var first = await GenerateShardedTestFileNames(shardCount: 4, shardIndex: 1, fileCount: 40);
+        var second = await GenerateShardedTestFileNames(shardCount: 4, shardIndex: 1, fileCount: 40);
+        Assert.That(second, Is.EquivalentTo(first));
+    }
+
+    [Test]
+    public void InvalidShardCountThrows()
+    {
+        Assert.Throws<ArgumentException>(() =>
+            _ = new TestSuiteGenerator(new TestSuiteGeneratorOptions { ShardCount = 0 }, usedSettingsFilePath: null));
+    }
+
+    [Test]
+    public void ShardIndexOutOfRangeThrows()
+    {
+        Assert.Throws<ArgumentException>(() =>
+            _ = new TestSuiteGenerator(new TestSuiteGeneratorOptions { ShardCount = 4, ShardIndex = 4 }, usedSettingsFilePath: null));
+        Assert.Throws<ArgumentException>(() =>
+            _ = new TestSuiteGenerator(new TestSuiteGeneratorOptions { ShardCount = 4, ShardIndex = -1 }, usedSettingsFilePath: null));
+    }
+
+    private static MemoryFileSystem CreateShardingFixture(int fileCount)
+    {
+        var fs = new MemoryFileSystem();
+
+        fs.CreateDirectory("/harness");
+        fs.WriteAllText("/harness/assert.js", "// assert harness stub\n");
+        fs.WriteAllText("/harness/sta.js", "// sta harness stub\n");
+
+        fs.CreateDirectory("/test/built-ins/Sample");
+        for (var i = 0; i < fileCount; i++)
+        {
+            fs.WriteAllText(
+                $"/test/built-ins/Sample/test{i}.js",
+                $"/*---\ndescription: sample {i}\n---*/\n// body {i}\n");
+        }
+
+        return fs;
+    }
+
+    private static async Task<HashSet<string>> GenerateShardedTestFileNames(int shardCount, int shardIndex, int fileCount)
+    {
+        var fs = CreateShardingFixture(fileCount);
+        var stream = Test262Stream.FromFileSystem(fs, opts =>
+        {
+            opts.SubDirectories = ["built-ins"];
+            opts.GenerateInverseStrictTestCase = false;
+            opts.LogInfo = (_, _) => { };
+            opts.LogError = (_, _) => { };
+        });
+
+        var targetPath = Path.Combine(Path.GetTempPath(), "Test262HarnessGeneratorTests", Guid.NewGuid().ToString("N"));
+        Directory.CreateDirectory(targetPath);
+
+        try
+        {
+            var options = new TestSuiteGeneratorOptions
+            {
+                TargetPath = targetPath,
+                Namespace = "Generated.Tests",
+                SuiteGitSha = "stable-sha-for-snapshot",
+                ShardCount = shardCount,
+                ShardIndex = shardIndex,
+            };
+
+            var generator = new TestSuiteGenerator(options, usedSettingsFilePath: "stable-settings-path");
+            await generator.Generate(stream);
+
+            var file = Path.Combine(targetPath, "Tests262Harness.Tests.built-ins.generated.cs");
+
+            // A shard that received no files does not emit the sub-directory file at all.
+            if (!File.Exists(file))
+            {
+                return new HashSet<string>(StringComparer.Ordinal);
+            }
+
+            var content = await File.ReadAllTextAsync(file);
+            return ExtractTestCaseFileNames(content);
+        }
+        finally
+        {
+            Directory.Delete(targetPath, recursive: true);
+        }
+    }
+
+    private static HashSet<string> ExtractTestCaseFileNames(string generated)
+    {
+        var set = new HashSet<string>(StringComparer.Ordinal);
+        foreach (Match match in Regex.Matches(generated, "\\[TestCase\\(\"(?<file>[^\"]+)\""))
+        {
+            set.Add(match.Groups["file"].Value);
+        }
+
+        return set;
+    }
 }