From 1557f63ae0dbf94bfdc8fbb15b00c4330df58696 Mon Sep 17 00:00:00 2001 From: Marko Lahma Date: Mon, 13 Jul 2026 13:17:44 +0300 Subject: [PATCH] Add generate-time suite sharding (--shard-count / --shard-index) Splits the generated suite into N deterministic, disjoint slices so a CI matrix can compile and run them on separate runners in parallel. Assignment is a stable FNV-1a hash of the test file name (not string.GetHashCode, which is per-process randomized), so shard k always holds the same files across runs/machines/OSes, shards never overlap, and their union is the full suite. Strict and non-strict variants of a file share a shard. Wired through TestSuiteGeneratorOptions, the generator, and the console tool. Added partition/determinism/validation tests; existing snapshots unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 14 ++ src/Test262Harness.Console/Program.cs | 18 +++ .../TestSuiteGenerator.cs | 41 ++++++ .../TestSuiteGeneratorOptions.cs | 17 +++ .../TestSuiteGeneratorTests.cs | 125 ++++++++++++++++++ 5 files changed, 215 insertions(+) diff --git a/README.md b/README.md index 632b094..bd2a031 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,20 @@ Exclusion maps to setting `[Ignore]` attribute in test suite. Non-parallel marking maps to setting `[NonParallelizable]` on the affected generated test method(s). Because all tests under a given sub-directory (e.g. `built-ins/Atomics/waitAsync`) collapse into one generated method, marking any one entry serialises the whole group — sufficient for timing-sensitive feature suites such as `Atomics.waitAsync`. File entries use the test262 forward-slash path format; esprima-style `(default)` / `(strict mode)` suffixes are not supported here (the attribute is method-level). +## Sharding the generated suite + +`generate` accepts two command-line options for splitting the suite into independent slices so a CI matrix can compile and run them on separate runners in parallel: + +| Option | Default | Description | +|:-----------------|:--------|:----------------------------------------------------------------| +| `--shard-count` | 1 | Total number of shards to split the suite into (1 = no sharding) | +| `--shard-index` | 0 | Zero-based index of the shard to emit, in range `[0, shard-count)` | + +``` +dotnet test262 generate --shard-count 4 --shard-index 0 +``` + +Each test file is assigned to a shard by a stable, order-independent hash of its name, so the assignment is deterministic across runs, machines and operating systems: shard *k* always contains the same files, the shards never overlap, and their union is exactly the full (unsharded) suite. Both strict and non-strict variants of a file always land in the same shard. Because the assignment does not depend on where generation runs, a shard generated on one operating system can be reused (e.g. from a cross-OS cache) by test jobs on any other. ## Branches and releases diff --git a/src/Test262Harness.Console/Program.cs b/src/Test262Harness.Console/Program.cs index 3ef321a..1588f2e 100644 --- a/src/Test262Harness.Console/Program.cs +++ b/src/Test262Harness.Console/Program.cs @@ -51,6 +51,14 @@ public sealed class Settings : CommandSettings [CommandOption("-s|--settings")] [DefaultValue("Test262Harness.settings.json")] public string SettingsFile { get; set; } = ""; + + [Description("Total number of shards to split the suite into (default 1 = no sharding)")] + [CommandOption("--shard-count")] + public int? ShardCount { get; set; } + + [Description("Zero-based index of the shard to emit, in range [0, shard-count)")] + [CommandOption("--shard-index")] + public int? ShardIndex { get; set; } } protected override async Task ExecuteAsync([NotNull] CommandContext context, [NotNull] Settings settings, CancellationToken cancellationToken) @@ -152,6 +160,16 @@ private static async Task FinalizeOptions(Settings settings, TestSuiteGeneratorO options.ExcludedFilesSource = settings.ExcludedFilesSource; } + if (settings.ShardCount is not null) + { + options.ShardCount = settings.ShardCount.Value; + } + + if (settings.ShardIndex is not null) + { + options.ShardIndex = settings.ShardIndex.Value; + } + if (!string.IsNullOrWhiteSpace(options.ExcludedFilesSource)) { var lines = await File.ReadAllLinesAsync(options.ExcludedFilesSource); diff --git a/src/Test262Harness.Generator/TestSuiteGenerator.cs b/src/Test262Harness.Generator/TestSuiteGenerator.cs index 381a434..1b537dc 100644 --- a/src/Test262Harness.Generator/TestSuiteGenerator.cs +++ b/src/Test262Harness.Generator/TestSuiteGenerator.cs @@ -32,6 +32,16 @@ public class TestSuiteGenerator public TestSuiteGenerator(TestSuiteGeneratorOptions options, string? usedSettingsFilePath) { + if (options.ShardCount < 1) + { + throw new ArgumentException($"ShardCount must be >= 1 but was {options.ShardCount}.", nameof(options)); + } + + if (options.ShardIndex < 0 || options.ShardIndex >= options.ShardCount) + { + throw new ArgumentException($"ShardIndex must be in the range [0, {options.ShardCount}) but was {options.ShardIndex}.", nameof(options)); + } + _options = options; _usedSettingsFilePath = usedSettingsFilePath; @@ -116,6 +126,11 @@ [.. _options.NonParallelFiles { var tests = stream.GetTestFiles([item]).ToList(); + if (_options.ShardCount > 1) + { + tests = tests.Where(x => GetShardIndex(x.FileName) == _options.ShardIndex).ToList(); + } + if (tests.Count == 0) { continue; @@ -205,6 +220,32 @@ private void SetCommonInfo(TemplateContext context, string templateHash) context.SetValue("TemplateSha", templateHash); } + private int GetShardIndex(string fileName) + { + return (int) (StableHash(fileName) % (uint) _options.ShardCount); + } + + /// + /// A small, deterministic hash (FNV-1a, 32-bit) used to assign a test file to a shard. + /// Unlike — which is randomized per process on modern .NET — + /// this produces identical results across runs, machines and operating systems, so a given file + /// always lands in the same shard regardless of where generation happens. + /// + private static uint StableHash(string value) + { + const uint offsetBasis = 2166136261; + const uint prime = 16777619; + + var hash = offsetBasis; + foreach (var c in value) + { + hash = (hash ^ (byte) c) * prime; + hash = (hash ^ (byte) (c >> 8)) * prime; + } + + return hash; + } + private static string StripTestPrefix(string fileName) { return fileName.StartsWith(TestPrefix, StringComparison.OrdinalIgnoreCase) diff --git a/src/Test262Harness.Generator/TestSuiteGeneratorOptions.cs b/src/Test262Harness.Generator/TestSuiteGeneratorOptions.cs index e4c54a4..9e42f5d 100644 --- a/src/Test262Harness.Generator/TestSuiteGeneratorOptions.cs +++ b/src/Test262Harness.Generator/TestSuiteGeneratorOptions.cs @@ -25,4 +25,21 @@ public class TestSuiteGeneratorOptions public string[] NonParallelFiles { get; set; } = []; public string ExcludedFilesSource { get; set; } = ""; + + /// + /// Total number of shards to split the generated suite into. Defaults to 1 (no sharding). + /// When greater than 1, each generation only emits the test cases belonging to , + /// letting a CI matrix compile and run distinct slices of the suite in parallel. The assignment is a + /// stable, order-independent hash of the test file name, so the shards are deterministic across runs, + /// machines and operating systems, and their union is exactly the full (unsharded) suite. + /// + [JsonIgnore] + public int ShardCount { get; set; } = 1; + + /// + /// Zero-based index of the shard to emit when is greater than 1. + /// Must be in the range [0, ShardCount). + /// + [JsonIgnore] + public int ShardIndex { get; set; } } diff --git a/test/Test262Harness.Tests/TestSuiteGeneratorTests.cs b/test/Test262Harness.Tests/TestSuiteGeneratorTests.cs index 4f8b3e2..2835cf7 100644 --- a/test/Test262Harness.Tests/TestSuiteGeneratorTests.cs +++ b/test/Test262Harness.Tests/TestSuiteGeneratorTests.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; using Test262Harness.TestSuite.Generator; using Zio; using Zio.FileSystems; @@ -140,4 +141,128 @@ public async Task IgnoredGroupDoesNotGetNonParallelizable() }); await VerifyOutput(output); } + + [Test] + public async Task ShardingPartitionsSuiteWithoutOverlapOrLoss() + { + const int fileCount = 40; + const int shardCount = 4; + + var all = await GenerateShardedTestFileNames(shardCount: 1, shardIndex: 0, fileCount); + Assert.That(all, Has.Count.EqualTo(fileCount)); + + var union = new HashSet(StringComparer.Ordinal); + var perShardCounts = new List(); + for (var shardIndex = 0; shardIndex < shardCount; shardIndex++) + { + var shard = await GenerateShardedTestFileNames(shardCount, shardIndex, fileCount); + perShardCounts.Add(shard.Count); + foreach (var fileName in shard) + { + Assert.That(union.Add(fileName), Is.True, $"File {fileName} appeared in more than one shard"); + } + } + + Assert.That(union, Is.EquivalentTo(all), "Union of all shards must equal the full (unsharded) suite"); + Assert.That(perShardCounts, Has.All.GreaterThan(0), "Every shard should receive at least one test"); + } + + [Test] + public async Task ShardAssignmentIsDeterministic() + { + var first = await GenerateShardedTestFileNames(shardCount: 4, shardIndex: 1, fileCount: 40); + var second = await GenerateShardedTestFileNames(shardCount: 4, shardIndex: 1, fileCount: 40); + Assert.That(second, Is.EquivalentTo(first)); + } + + [Test] + public void InvalidShardCountThrows() + { + Assert.Throws(() => + _ = new TestSuiteGenerator(new TestSuiteGeneratorOptions { ShardCount = 0 }, usedSettingsFilePath: null)); + } + + [Test] + public void ShardIndexOutOfRangeThrows() + { + Assert.Throws(() => + _ = new TestSuiteGenerator(new TestSuiteGeneratorOptions { ShardCount = 4, ShardIndex = 4 }, usedSettingsFilePath: null)); + Assert.Throws(() => + _ = new TestSuiteGenerator(new TestSuiteGeneratorOptions { ShardCount = 4, ShardIndex = -1 }, usedSettingsFilePath: null)); + } + + private static MemoryFileSystem CreateShardingFixture(int fileCount) + { + var fs = new MemoryFileSystem(); + + fs.CreateDirectory("/harness"); + fs.WriteAllText("/harness/assert.js", "// assert harness stub\n"); + fs.WriteAllText("/harness/sta.js", "// sta harness stub\n"); + + fs.CreateDirectory("/test/built-ins/Sample"); + for (var i = 0; i < fileCount; i++) + { + fs.WriteAllText( + $"/test/built-ins/Sample/test{i}.js", + $"/*---\ndescription: sample {i}\n---*/\n// body {i}\n"); + } + + return fs; + } + + private static async Task> GenerateShardedTestFileNames(int shardCount, int shardIndex, int fileCount) + { + var fs = CreateShardingFixture(fileCount); + var stream = Test262Stream.FromFileSystem(fs, opts => + { + opts.SubDirectories = ["built-ins"]; + opts.GenerateInverseStrictTestCase = false; + opts.LogInfo = (_, _) => { }; + opts.LogError = (_, _) => { }; + }); + + var targetPath = Path.Combine(Path.GetTempPath(), "Test262HarnessGeneratorTests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(targetPath); + + try + { + var options = new TestSuiteGeneratorOptions + { + TargetPath = targetPath, + Namespace = "Generated.Tests", + SuiteGitSha = "stable-sha-for-snapshot", + ShardCount = shardCount, + ShardIndex = shardIndex, + }; + + var generator = new TestSuiteGenerator(options, usedSettingsFilePath: "stable-settings-path"); + await generator.Generate(stream); + + var file = Path.Combine(targetPath, "Tests262Harness.Tests.built-ins.generated.cs"); + + // A shard that received no files does not emit the sub-directory file at all. + if (!File.Exists(file)) + { + return new HashSet(StringComparer.Ordinal); + } + + var content = await File.ReadAllTextAsync(file); + return ExtractTestCaseFileNames(content); + } + finally + { + Directory.Delete(targetPath, recursive: true); + } + } + + private static HashSet ExtractTestCaseFileNames(string generated) + { + var set = new HashSet(StringComparer.Ordinal); + foreach (Match match in Regex.Matches(generated, "\\[TestCase\\(\"(?[^\"]+)\"")) + { + set.Add(match.Groups["file"].Value); + } + + return set; + } }