From 678613b414becc55f0750cadfa26d42510df23ba Mon Sep 17 00:00:00 2001 From: Marko Lahma Date: Sun, 23 Aug 2026 11:18:42 +0300 Subject: [PATCH] Wpt: derive the corpus inventory table instead of maintaining it by hand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #3251 asked whether the WPT driver should get a Test262Harness-style tool, and required Phase 0 to try to kill the proposal first. Three of its four kill criteria fire; only the census survives, so only the census is built here. Frequency (criterion 1) says the whole history of Jint.Tests/Wpt is 27 commits across three days, with ten corpus-sync events all inside one campaign that has ended — so sync is burst-shaped and wants a recipe, not a tool. A shell recipe (criterion 2) is genuinely sufficient for the sync half: the byte-verification is 20 lines, it runs in under a minute over 48 API calls, and running it now reports 359 files with no drift at this pin. Prior art (criterion 4) would be a downgrade rather than a subsumption: Deno's expectation.json carries no reasoning at all — its schema forbids extra keys and `update` overwrites leaves — and Node's status files enforce only `fail.expected`, leaving `skip`, the majority of their entries, never checked against reality. Neither carries a divergence category. Jint's exclusion table is two-sided over every entry and mandates a category plus prose, so it is left exactly as it is. The staleness the census closes is real and is not tied to corpus additions: of the six times the table moved, five came from engine-fix commits, and two commits fixed WPT-found defects without touching it at all. Every figure in it is arithmetic the driver already does. So the table becomes a derived artifact, in two halves split by cost: - The standards, suite counts and file counts are read off the embedded corpus and are checked always, on every platform and in every filtered run, in 45 ms. This is also what refuses a corpus vendored with no row of its own: the census prefixes must partition every vendored .any.js. - The assertion and not-passing totals need the suites to have run. The check reuses every outcome the driver already produced — WptHarness.Run hands each file's results to WptCensus on the way back out — but always-on it still measured Jint.Tests at 65 s -> 85 s, which is material, so it is gated on JINT_WPT_CENSUS. The PR workflow's Windows leg sets it, in the same shape as the host-contract-verification leg beside it; an ordinary run pays nothing. JINT_WPT_CENSUS=update rewrites the table instead of failing against it, which replaces the hand re-census that had to be done four times. WptTestRunner's theories, the shim and the exclusion table's runtime enforcement are untouched. The corpus is unchanged: no vendored file is modified and no row of the table moves. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FHnbgQGW5QgfatxL6DgGEd --- .github/workflows/pr.yml | 9 +- Jint.Tests/Jint.Tests.csproj | 4 + Jint.Tests/Wpt/Vendor/README.md | 71 ++++- Jint.Tests/Wpt/WptCensus.cs | 473 +++++++++++++++++++++++++++++++ Jint.Tests/Wpt/WptCensusTests.cs | 117 ++++++++ Jint.Tests/Wpt/WptHarness.cs | 8 +- 6 files changed, 677 insertions(+), 5 deletions(-) create mode 100644 Jint.Tests/Wpt/WptCensus.cs create mode 100644 Jint.Tests/Wpt/WptCensusTests.cs diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6512002a58..62f16d9e1d 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -29,11 +29,18 @@ jobs: os: - { name: 'linux', image: 'ubuntu-latest' } - { name: 'linux - ARM', image: 'ubuntu-24.04-arm' } - - { name: 'windows', image: 'windows-latest' } + # The WPT corpus inventory in Jint.Tests/Wpt/Vendor/README.md is measured on Windows — assertion + # counts move per operating system — so this is the one leg entitled to check it. The census reuses + # the outcomes the suites produce anyway and cost 65 s -> 85 s of Jint.Tests when it was measured + # always-on, which is why it is opt-in everywhere else rather than part of an ordinary run. + - { name: 'windows', image: 'windows-latest', census: '1' } - { name: 'macos', image: 'macos-latest' } runs-on: ${{ matrix.os.image }} + env: + JINT_WPT_CENSUS: ${{ matrix.os.census }} + steps: - name: Setup NET diff --git a/Jint.Tests/Jint.Tests.csproj b/Jint.Tests/Jint.Tests.csproj index a6453ee917..97b94eb88d 100644 --- a/Jint.Tests/Jint.Tests.csproj +++ b/Jint.Tests/Jint.Tests.csproj @@ -34,6 +34,10 @@ wpt-prelude/%(Filename)%(Extension) + + diff --git a/Jint.Tests/Wpt/Vendor/README.md b/Jint.Tests/Wpt/Vendor/README.md index f1fb3d846b..8f6340ef78 100644 --- a/Jint.Tests/Wpt/Vendor/README.md +++ b/Jint.Tests/Wpt/Vendor/README.md @@ -872,6 +872,10 @@ citation and an argued decision, never a to-do — and the rule that age alone n ## The whole corpus, standard by standard +**This table is generated, not maintained.** `Jint.Tests/Wpt/WptCensusTests.cs` derives every figure in it +from the corpus and fails when the README disagrees — see [Taking the census](#taking-the-census) below for +the two halves of that check and the one command that rewrites the table. + Measured at this pin, on Windows, with the driver's exclusion table in force. "Not passing" is every result the shim did not record `PASS`, which is exactly the set the table names. Re-measured in full for [#3212](https://github.com/sebastienros/jint/issues/3212), which is why five rows move at once: the Fetch row @@ -899,8 +903,8 @@ Re-censused whole rather than adjusted row by row, because several rows had gone that moved them: before [#3195](https://github.com/sebastienros/jint/issues/3195) the true figures were 270 files / 40,631 assertions / 2,907 not passing, where this table read 269 / 40,617 / 2,980 — and Streams (16 against a real 11), Compression (84 against 22), User Timing (9 against 5) and DOM (2 against 0) were each -carrying a number a later fix had already improved. Take the census again when a change moves any of them; it -is one run of the driver with the results tallied per directory. +carrying a number a later fix had already improved. That class of drift is what the census closes: those four +rows were arithmetic the driver already did, and nothing checked the prose against it. Two of those rows are worth a caveat. The Encoding figure is dominated by `textdecoder-fatal-single-byte.any.js`, 7,168 assertions of it and every one passing; of the 322 that do not, @@ -930,8 +934,69 @@ directories with no vendored file at all are `fetch/api/request/` (no API base U above, the `WebCryptoAPI` directories listed above, and `xhr/` (there is no `XMLHttpRequest` in the engine — the shim's is a vendored-corpus reader for the suites that need one, never an implementation). +## Taking the census + +The table above is derived from the corpus by `Jint.Tests/Wpt/WptCensusTests.cs`, in two halves split by what +each column costs to know. + +`TheInventoryTableNamesEveryStandardAndCountsItsFilesAndSuites` holds the **Standard**, **Suites** and +**Files** columns. Those are read off the embedded corpus — a row's files are its directory's `.any.js` files, +and its suites are the directories they sit in — so it needs no execution, runs on every platform and in every +filtered run, and costs a few milliseconds. It is also what catches a corpus arriving with no row of its own: +the census prefixes have to partition every vendored `.any.js`, so vendoring a new standard fails this test +until the standard is named. + +`TheInventoryTableMatchesWhatTheCorpusMeasures` holds all five columns, which means running the suites. It is +**opt-in**, because it is the only part that costs anything: it reuses every outcome the driver already +produced — `WptHarness.Run` hands each file's results to `WptCensus` on the way back out — but a full +`Jint.Tests` pass still measured 65 s → 85 s with it always on. The PR workflow's Windows leg sets the +variable, so that is where the table is enforced; an ordinary run pays nothing. Windows only, in both modes, +because assertion counts move per operating system — the Web Cryptography row most of all — and this table is +measured on Windows. + +```bash +# check the table (about a minute; it censuses the whole corpus) +JINT_WPT_CENSUS=1 dotnet test -c Release Jint.Tests/Jint.Tests.csproj -f net10.0 \ + --filter "FullyQualifiedName~Jint.Tests.Wpt.WptCensusTests" + +# rewrite the table from what the corpus measures, then commit the diff +JINT_WPT_CENSUS=update dotnet test -c Release Jint.Tests/Jint.Tests.csproj -f net10.0 \ + --filter "FullyQualifiedName~Jint.Tests.Wpt.WptCensusTests" +``` + +A failure prints the stated table and the measured one side by side, so the rows that moved are the rows that +differ. + ## Updating the pin Resolve `master` to a concrete commit, re-fetch every file in this directory at that commit, update the SHA above, and run the suites. Expect the exclusion table to need work in the same change: the driver fails on an -entry that no longer applies, which is the point. +entry that no longer applies, which is the point. Re-run the census afterwards — a bump moves nearly every row +of the table above. + +### Verifying that nothing has drifted + +Vendored files are copied verbatim, and the failure mode is silent: a file that has quietly diverged from the +pin still runs, still passes, and stops testing what upstream asserts. Checking is mechanical, and needs no +downloads — a GitHub blob id *is* `git hash-object` of the content, and `.gitattributes` pins this whole +directory to `text eol=lf` precisely so a Windows checkout cannot make the two disagree. + +```bash +cd Jint.Tests/Wpt/Vendor +SHA=$(grep -oE '\b[0-9a-f]{40}\b' README.md | head -1) + +# one call per directory that holds a vendored file (48 at this pin) +for d in $(find . -type f \( -name '*.js' -o -name '*.json' \) -printf '%h\n' | sort -u | sed 's|^\./||'); do + gh api "repos/web-platform-tests/wpt/contents/$d?ref=$SHA" \ + --jq '.[] | select(.type=="file") | "\(.sha) \(.path)"' +done > /tmp/upstream.txt + +find . -type f \( -name '*.js' -o -name '*.json' \) | sort | while read -r f; do + rel="${f#./}" + up=$(grep -m1 " ${rel}$" /tmp/upstream.txt | cut -d' ' -f1) + [ -z "$up" ] && { echo "NOT-IN-UPSTREAM: $rel"; continue; } + [ "$(git hash-object "$f")" = "$up" ] || echo "DRIFT: $rel" +done +``` + +Silence is a clean corpus. Verified at this pin: 359 files, no drift, no file absent upstream. diff --git a/Jint.Tests/Wpt/WptCensus.cs b/Jint.Tests/Wpt/WptCensus.cs new file mode 100644 index 0000000000..785f7f78be --- /dev/null +++ b/Jint.Tests/Wpt/WptCensus.cs @@ -0,0 +1,473 @@ +#if NET8_0_OR_GREATER +#nullable enable + +using System.Collections.Concurrent; +using System.Globalization; +using System.Text; + +namespace Jint.Tests.Wpt; + +/// +/// The corpus inventory table in Vendor/README.md — "The whole corpus, standard by standard" — as a +/// derived artifact rather than as prose somebody keeps up to date by hand. +/// +/// +/// +/// Every figure in that table is arithmetic the driver already does. A row's Files is the +/// vendored .any.js count under its directory, its Suites is how many theories cover them, and +/// its Assertions and Not passing are the totals the shim reported while the suite ran. Left as +/// prose, all four drift the moment a fix removes an exclusion or a corpus grows a file, and the drift is +/// invisible: nothing goes red, the table just quietly describes an engine that no longer exists. It has +/// happened — the README's own note beside the table records four rows carrying a number a later fix had +/// already improved, and a re-census correcting 269/40,617/2,980 to 270/40,631/2,907 in one go. +/// +/// +/// What is checked when splits along what each column depends on. +/// holds the +/// Standard, Suites and Files columns — those three are read off the embedded corpus and +/// need no execution at all, so it runs on every platform, in a filtered run, and for free. +/// holds all five, which needs the +/// suites to have run. It runs on Windows only: assertion counts move per platform — the WebCryptoAPI row most +/// of all, which is what the per-OS exclusion scoping exists for — and the table states in its own first line +/// that it is measured on Windows. +/// +/// +/// The measured check pays for at most one pass over the corpus, and usually much less. Every outcome +/// the theories produce is handed here on the way back out of , so a file the +/// driver has already run is tallied rather than re-run. The two classes are separate collections and xUnit +/// runs them in parallel, so in a full Jint.Tests pass most of the corpus is already recorded by the +/// time the census reaches it. What it cannot reuse it runs itself, which is what makes the check work as a +/// standalone command as well: filter to that class and it censuses the corpus from cold. +/// +/// +/// It is still opt-in, because "much less" is not nothing: always-on, it measured a full +/// Jint.Tests pass at 65 s → 85 s. So it is gated on — 1 checks the +/// table, update rewrites it — and the PR workflow's Windows leg is what sets it, in the same shape as +/// the JINT_HOST_CONTRACT_VERIFICATION leg beside it. An ordinary developer run pays only the free +/// half. +/// +/// +internal static class WptCensus +{ + /// + /// The environment variable that turns the check into a rewrite. + /// + internal const string UpdateVariable = "JINT_WPT_CENSUS"; + + /// + /// The README, embedded so that verification reads the same bytes the build compiled and needs no source + /// tree at all. Only the rewrite goes looking for the file on disk. + /// + private const string ReadmeResourceName = "wpt-vendor-readme.md"; + + private const string TableHeading = "## The whole corpus, standard by standard"; + private const string TableHeader = "| Standard | Suites | Files | Assertions | Not passing |"; + private const string TableDivider = "| --- | --- | --- | --- | --- |"; + + /// + /// Which README row a vendored path belongs to, in the order the table lists them. + /// + /// + /// This is the one editorial half of the census: a directory prefix carries no standard's name, and two + /// of the rows deliberately group several ("HTML" is three html/webappapis/ suites in one row and + /// workers/ in another). The prefixes must still partition every vendored .any.js exactly — + /// a corpus arriving under a directory no row claims fails the check rather than going uncounted, which is + /// what stops a new standard being vendored with no row of its own. + /// + private static readonly (string Standard, string Prefix)[] _standards = + [ + ("URL", "url/"), + ("URL Pattern", "urlpattern/"), + ("Encoding", "encoding/"), + ("Web Cryptography", "WebCryptoAPI/"), + ("Streams", "streams/"), + ("Compression", "compression/"), + ("File API", "FileAPI/"), + ("High Resolution Time", "hr-time/"), + ("User Timing", "user-timing/"), + ("HTML — workers", "workers/"), + ("HTML — timers, microtasks, structured clone", "html/webappapis/"), + ("DOM", "dom/"), + ("Fetch", "fetch/api/"), + ]; + + private static readonly ConcurrentDictionary _observed = new(StringComparer.Ordinal); + + /// + /// What one file contributed to its standard's row. + /// + private readonly record struct Counts(int Assertions, int NotPassing); + + /// + /// One line of the table. and are read off the corpus; + /// the other two are only meaningful once every file of the row has run. + /// + private readonly record struct Row( + string Standard, + string Prefix, + int Suites, + int Files, + int Assertions, + int NotPassing); + + /// + /// Records what a suite file produced. Called from , which is the single + /// funnel every vendored file's outcome comes back through — so the census sees the whole corpus without + /// the theories knowing it exists, and re-running one file simply overwrites its own entry. + /// + internal static void Record(string testFilePath, WptRunOutcome outcome) + { + var notPassing = 0; + foreach (var result in outcome.Results) + { + if (!result.Passed) + { + notPassing++; + } + } + + _observed[testFilePath] = new Counts(outcome.Results.Count, notPassing); + } + + /// + /// Every vendored .any.js, which is one theory case each and one row's worth of files. + /// + internal static List VendoredTestFiles() + { + var files = new List(); + foreach (var path in WptCorpus.Paths) + { + if (path.EndsWith(".any.js", StringComparison.Ordinal)) + { + files.Add(path); + } + } + + files.Sort(StringComparer.Ordinal); + return files; + } + + /// + /// Makes sure every vendored file has reported, running only the ones the driver has not already run. + /// + /// How many files this had to run itself, which is what the census actually cost. + /// + /// A file the theories already covered is tallied from what they produced, so the two chains overlap + /// rather than duplicate. Racing a theory on the same file is harmless: a suite file's outcome does not + /// depend on when it ran — nothing in the corpus is timing-dependent, which is the same property the + /// harness deadline relies on — so both writers record the same pair. + /// + internal static int Measure() + { + var ran = 0; + foreach (var file in VendoredTestFiles()) + { + if (!_observed.ContainsKey(file)) + { + // Records itself on the way back out. + WptHarness.Run(file); + ran++; + } + } + + return ran; + } + + /// + /// Renders the table. false leaves the two counted columns out of the + /// comparison by rendering whatever the README already claims for them, so the caller can hold the three + /// derived columns without pretending to know the other two. + /// + internal static string Render(bool measured, IReadOnlyList? readmeLines = null) + { + var claimed = measured ? null : ParseClaimedCounts(readmeLines ?? ReadReadme().Split('\n')); + + var files = VendoredTestFiles(); + var seen = new HashSet(StringComparer.Ordinal); + var rows = new List(); + + foreach (var (standard, prefix) in _standards) + { + var suites = new HashSet(StringComparer.Ordinal); + var count = 0; + var assertions = 0; + var notPassing = 0; + + foreach (var file in files) + { + if (!file.StartsWith(prefix, StringComparison.Ordinal)) + { + continue; + } + + seen.Add(file); + + // A suite is a directory, because WptCorpus.TestFiles lists a directory's own files and never + // descends — so the theory count of a standard is the number of directories its files sit in. + suites.Add(WptCorpus.DirectoryOf(file)); + count++; + + if (measured) + { + var counts = _observed[file]; + assertions += counts.Assertions; + notPassing += counts.NotPassing; + } + } + + if (!measured && claimed!.TryGetValue(standard, out var already)) + { + (assertions, notPassing) = already; + } + + rows.Add(new Row(standard, prefix, suites.Count, count, assertions, notPassing)); + } + + var unclaimed = new List(); + foreach (var file in files) + { + if (!seen.Contains(file)) + { + unclaimed.Add(file); + } + } + + if (unclaimed.Count > 0) + { + throw new InvalidOperationException( + "These vendored files belong to no census row, so no line of the inventory table counts them. " + + $"Add the standard to {nameof(WptCensus)}.{nameof(_standards)} and give it a row:" + + Environment.NewLine + + string.Join(Environment.NewLine, unclaimed)); + } + + return RenderTable(rows); + } + + private static string RenderTable(List rows) + { + var table = new StringBuilder(); + table.Append(TableHeader).Append('\n'); + table.Append(TableDivider).Append('\n'); + + var suites = 0; + var files = 0; + var assertions = 0; + var notPassing = 0; + + foreach (var row in rows) + { + var scope = row.Suites == 1 ? $"`{row.Prefix}`" : $"`{row.Prefix}` ×{row.Suites}"; + table.Append( + $"| {row.Standard} | {scope} | {Number(row.Files)} | {Number(row.Assertions)} | {Number(row.NotPassing)} |") + .Append('\n'); + + suites += row.Suites; + files += row.Files; + assertions += row.Assertions; + notPassing += row.NotPassing; + } + + table.Append( + $"| **total** | **{Number(suites)}** | **{Number(files)}** | **{Number(assertions)}** | **{Number(notPassing)}** |") + .Append('\n'); + + return table.ToString(); + } + + private static string Number(int value) => value.ToString("N0", CultureInfo.InvariantCulture); + + /// + /// The two measured columns as the README currently states them, keyed by standard, so a check that is + /// not entitled to an opinion on them can carry them through unchanged. + /// + private static Dictionary ParseClaimedCounts(IReadOnlyList lines) + { + var claimed = new Dictionary(StringComparer.Ordinal); + + foreach (var line in TableLines(lines)) + { + var cells = Cells(line); + if (cells.Length != 5 || cells[0].StartsWith("**", StringComparison.Ordinal) || cells[0] == "Standard" + || cells[0].StartsWith("---", StringComparison.Ordinal)) + { + continue; + } + + if (TryParseNumber(cells[3], out var assertions) && TryParseNumber(cells[4], out var notPassing)) + { + claimed[cells[0]] = (assertions, notPassing); + } + } + + return claimed; + } + + private static bool TryParseNumber(string cell, out int value) => + int.TryParse(cell.Replace(",", "").Trim('*'), NumberStyles.Integer, CultureInfo.InvariantCulture, out value); + + private static string[] Cells(string line) => + Array.ConvertAll(line.Trim().Trim('|').Split('|'), cell => cell.Trim()); + + /// + /// The table's own lines, which are every consecutive pipe-prefixed line from the header on. The heading + /// above it is what makes the search unambiguous — the README carries a second table, of what is + /// deliberately not vendored. + /// + private static IEnumerable TableLines(IReadOnlyList lines) + { + var (start, end) = Locate(lines); + for (var i = start; i < end; i++) + { + yield return lines[i]; + } + } + + /// + /// Where the table sits in the README, as a half-open line range. + /// + internal static (int Start, int End) Locate(IReadOnlyList lines) + { + var heading = -1; + for (var i = 0; i < lines.Count; i++) + { + if (string.Equals(lines[i].TrimEnd('\r'), TableHeading, StringComparison.Ordinal)) + { + heading = i; + break; + } + } + + if (heading < 0) + { + throw new InvalidOperationException( + $"Vendor/README.md no longer has a \"{TableHeading}\" section, which is where the census writes."); + } + + for (var i = heading; i < lines.Count; i++) + { + if (!string.Equals(lines[i].TrimEnd('\r'), TableHeader, StringComparison.Ordinal)) + { + continue; + } + + var end = i; + while (end < lines.Count && lines[end].StartsWith("|", StringComparison.Ordinal)) + { + end++; + } + + return (i, end); + } + + throw new InvalidOperationException( + $"Vendor/README.md's \"{TableHeading}\" section no longer holds a \"{TableHeader}\" table."); + } + + /// + /// The inventory table exactly as the README states it, normalised to LF and ending in one newline. + /// + internal static string ReadmeTable(IReadOnlyList? lines = null) + { + lines ??= ReadReadme().Split('\n'); + + var table = new StringBuilder(); + foreach (var line in TableLines(lines)) + { + table.Append(line.TrimEnd('\r')).Append('\n'); + } + + return table.ToString(); + } + + internal static string ReadReadme() + { + var assembly = typeof(WptCensus).Assembly; + using var stream = assembly.GetManifestResourceStream(ReadmeResourceName) + ?? throw new FileNotFoundException($"Embedded resource \"{ReadmeResourceName}\" is missing.", ReadmeResourceName); + using var reader = new StreamReader(stream, Encoding.UTF8); + return reader.ReadToEnd(); + } + + /// + /// Whether asked for the census at all — 1, true or + /// update. Unset is the default, and the reason the measured check costs a normal run nothing. + /// + internal static bool CensusRequested() + { + var value = Environment.GetEnvironmentVariable(UpdateVariable); + return string.Equals(value, "1", StringComparison.Ordinal) + || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "update", StringComparison.OrdinalIgnoreCase); + } + + /// + /// Whether it asked for the table to be rewritten rather than checked. + /// + internal static bool UpdateRequested() => + string.Equals(Environment.GetEnvironmentVariable(UpdateVariable), "update", StringComparison.OrdinalIgnoreCase); + + /// + /// Writes the rendered table over the one in the working tree's Vendor/README.md, preserving the + /// file's own line endings. + /// + internal static string WriteReadmeTable(string table) + { + var path = LocateReadmeOnDisk(); + var original = File.ReadAllText(path); + var crlf = original.Contains("\r\n", StringComparison.Ordinal); + var lines = original.Split('\n'); + + var (start, end) = Locate(lines); + var rebuilt = new List(lines.Length); + rebuilt.AddRange(lines[..start]); + rebuilt.AddRange(table.TrimEnd('\n').Split('\n')); + rebuilt.AddRange(lines[end..]); + + var joined = string.Join("\n", rebuilt.ConvertAll(line => line.TrimEnd('\r'))); + File.WriteAllText(path, crlf ? joined.Replace("\n", "\r\n") : joined, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + return path; + } + + /// + /// Finds the README in the working tree. The test runs out of artifacts/bin/…, so this walks up + /// looking for the repository rather than assuming a relative depth. + /// + private static string LocateReadmeOnDisk() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var candidate = Path.Combine(directory.FullName, "Jint.Tests", "Wpt", "Vendor", "README.md"); + if (File.Exists(candidate)) + { + return candidate; + } + + directory = directory.Parent; + } + + throw new InvalidOperationException( + $"{UpdateVariable} asked the census to rewrite Vendor/README.md, but no source tree was found above " + + $"\"{AppContext.BaseDirectory}\". Run the suites from the repository to update the table."); + } + + /// + /// The message a mismatch reports: the two tables side by side, and how to rewrite it. + /// + internal static string Explain(string expected, string actual, string what) + { + var message = new StringBuilder(); + message.Append("Vendor/README.md's inventory table is out of date (").Append(what).AppendLine(")."); + message.AppendLine(); + message.AppendLine("It says:"); + message.AppendLine(actual.TrimEnd('\n')); + message.AppendLine(); + message.AppendLine("The corpus says:"); + message.AppendLine(expected.TrimEnd('\n')); + message.AppendLine(); + message.Append($"Rewrite it by running the WPT suites with {UpdateVariable}=update set."); + return message.ToString(); + } +} +#endif diff --git a/Jint.Tests/Wpt/WptCensusTests.cs b/Jint.Tests/Wpt/WptCensusTests.cs new file mode 100644 index 0000000000..289116dba4 --- /dev/null +++ b/Jint.Tests/Wpt/WptCensusTests.cs @@ -0,0 +1,117 @@ +#if NET8_0_OR_GREATER +#nullable enable + +namespace Jint.Tests.Wpt; + +/// +/// Holds Vendor/README.md's corpus inventory — "The whole corpus, standard by standard" — to what the +/// corpus actually is, so the table is a derived artifact rather than prose somebody keeps up to date by hand. +/// +/// +/// +/// Two tests, split by what each column of that table costs to know. The standards, their suite counts and +/// their file counts are read straight off the embedded corpus and are checked always. The assertion and +/// not-passing totals need the suites to have run, so they are checked once the corpus has reported — see +/// for how that is paid for, and why it is nearly free in a full run. +/// +/// +/// Neither test touches the driver's rules. The exclusion table still decides what is forgiven and a failing +/// test that no entry names still fails its own suite in , whatever these say. +/// +/// +public class WptCensusTests +{ + /// + /// The inventory table must name every standard the corpus holds, and its Suites and Files + /// columns must be what is actually vendored. + /// + /// + /// + /// These three columns are derived without running anything — a row's files are its directory's + /// .any.js files, and its suites are the directories those files sit in, because + /// lists a directory's own files and never descends. So this holds on + /// every platform and in every filtered run, for the cost of reading an embedded string. + /// + /// + /// It is also what catches a corpus arriving with no row of its own: the census prefixes have to partition + /// every vendored .any.js, so a new standard fails here rather than being silently left out of the + /// inventory. The two counted columns are carried through from what the README already claims, so this + /// test cannot fail for them — but the total row is recomputed from the rows above it, which means a table + /// whose rows and total disagree fails here too. + /// + /// + [Fact] + public void TheInventoryTableNamesEveryStandardAndCountsItsFilesAndSuites() + { + var lines = WptCensus.ReadReadme().Split('\n'); + var stated = WptCensus.ReadmeTable(lines); + var derived = WptCensus.Render(measured: false, lines); + + if (!string.Equals(derived, stated, StringComparison.Ordinal)) + { + Assert.Fail(WptCensus.Explain(derived, stated, "the standards, suite counts or file counts have moved")); + } + } + + /// + /// The whole inventory table, including the assertion and not-passing totals, must be what the corpus + /// measures right now. + /// + /// + /// + /// This is the check the table has never had. Every figure in it is arithmetic the driver already + /// does, and left as prose it drifts silently: a fix that removes an exclusion moves a row, nothing goes + /// red, and the table quietly describes an engine that no longer exists. The README records exactly that + /// happening — four rows at once carrying a number a later fix had already improved, corrected only when + /// somebody re-censused the whole corpus by hand. + /// + /// + /// Opt-in, because it is the one part of the census that costs anything. The totals can only be + /// known by running the corpus, and although this reuses every outcome the driver already produced, a full + /// Jint.Tests pass still measured 65 s → 85 s with it always on. So it is gated on + /// JINT_WPT_CENSUS and a developer's ordinary run pays nothing; the PR workflow's Windows leg sets + /// the variable, which is where the table is actually enforced. That mirrors the + /// JINT_HOST_CONTRACT_VERIFICATION leg beside it. + /// + /// + /// JINT_WPT_CENSUS=1 checks the table; JINT_WPT_CENSUS=update rewrites it from the run. The + /// second is the whole maintenance procedure, and it replaces the hand re-census. + /// + /// + /// Windows only, in both modes, because assertion counts move per operating system and the table says in + /// its own first line that it is measured on Windows — updating it from a Linux run would write figures + /// the table does not claim. That is a deliberate hole: the three columns above are platform-independent + /// and are held everywhere. + /// + /// + [Fact] + public void TheInventoryTableMatchesWhatTheCorpusMeasures() + { + Assert.SkipUnless( + WptCensus.CensusRequested(), + $"the census runs the corpus to total it, so it is opt-in: set {WptCensus.UpdateVariable}=1 to " + + $"check the inventory table, or {WptCensus.UpdateVariable}=update to rewrite it."); + + Assert.SkipUnless( + OperatingSystem.IsWindows(), + "the inventory table is measured on Windows, and assertion counts move per platform."); + + var updating = WptCensus.UpdateRequested(); + WptCensus.Measure(); + + var measured = WptCensus.Render(measured: true); + + if (updating) + { + var path = WptCensus.WriteReadmeTable(measured); + Assert.Skip($"{WptCensus.UpdateVariable} rewrote the inventory table in {path}."); + } + + var stated = WptCensus.ReadmeTable(); + if (!string.Equals(measured, stated, StringComparison.Ordinal)) + { + Assert.Fail(WptCensus.Explain(measured, stated, "the corpus was censused and the totals disagree")); + } + } +} +#endif diff --git a/Jint.Tests/Wpt/WptHarness.cs b/Jint.Tests/Wpt/WptHarness.cs index f135777f6d..bab11a9808 100644 --- a/Jint.Tests/Wpt/WptHarness.cs +++ b/Jint.Tests/Wpt/WptHarness.cs @@ -142,9 +142,15 @@ internal static class WptHarness internal static WptRunOutcome Run(string testFilePath) { var directory = WptCorpus.DirectoryOf(testFilePath); - return RunsInAWorker(testFilePath) + var outcome = RunsInAWorker(testFilePath) ? RunInWorker(testFilePath, directory) : Execute(directory, MetaScripts(testFilePath, directory), WptCorpus.Read(testFilePath), testFilePath); + + // The corpus inventory in Vendor/README.md is tallied from the run rather than maintained by hand, and + // this is the one place every vendored file's outcome comes back through. Recording it here rather + // than in the theory keeps the census out of the driver's own rules entirely — see WptCensus. + WptCensus.Record(testFilePath, outcome); + return outcome; } ///