Shard Test262 across parallel CI runners + fix cross-OS suite cache

.gitattributes

@@ -13,3 +13,9 @@ Jint.Tests.Test262/** linguist-vendored
 *.verified.txt text eol=lf
 *.received.cs text eol=lf
 *.received.txt text eol=lf
+
+# The Test262 generated suite is cached across operating systems, keyed on a hash of this
+# settings file. It must therefore be byte-identical on every platform: a CRLF Windows checkout
+# would change both the cache key (hashFiles) and the MSBuild regeneration gate, turning every
+# cross-OS cache restore into a full regeneration.
+Jint.Tests.Test262/Test262Harness.settings.json text eol=lf

.github/workflows/build.yml

@@ -11,11 +11,16 @@ on:
 env:
   DOTNET_NOLOGO: true
   DOTNET_CLI_TELEMETRY_OPTOUT: 1
+  # Number of shards the Test262 suite is split into. Keep in sync with the `shard` matrix lists below.
+  TEST262_SHARD_COUNT: 4
 
 jobs:
-  build:
+
+  # Everything except Test262: fast, run once.
+  unit:
 
     runs-on: ubuntu-latest
+
     steps:
 
     - name: Setup NET
@@ -24,16 +29,110 @@ jobs:
         dotnet-version: 10.0
 
     - name: Checkout source code
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
+
+    - name: Test
+      shell: bash
+      run: |
+        set -e
+        for proj in \
+          Jint.Tests/Jint.Tests.csproj \
+          Jint.Tests.PublicInterface/Jint.Tests.PublicInterface.csproj \
+          Jint.Tests.CommonScripts/Jint.Tests.CommonScripts.csproj \
+          Jint.Tests.SourceGenerators/Jint.Tests.SourceGenerators.csproj; do
+          echo "::group::$proj"
+          dotnet test "$proj" --configuration Release \
+            --logger "console;verbosity=quiet" \
+            --logger "GitHubActions;summary-include-passed=false;summary-include-skipped=false"
+          echo "::endgroup::"
+        done
 
-    - name: Cache Test262 generated suite
-      uses: actions/cache@v5
+  # Generate each Test262 shard once and publish it to a cross-OS cache. Because default-branch
+  # caches are visible to pull requests, this seeds the shards that PR jobs restore.
+  generate-test262:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [0, 1, 2, 3]
+
+    runs-on: ubuntu-latest
+
+    steps:
+
+    - name: Setup NET
+      uses: actions/setup-dotnet@v5
+      with:
+        dotnet-version: 10.0
+
+    - name: Checkout source code
+      uses: actions/checkout@v7
+
+    - name: Cache Test262 generated shard
+      id: cache
+      uses: actions/cache@v6
       with:
         path: Jint.Tests.Test262/Generated
-        key: test262-generated-${{ hashFiles('Jint.Tests.Test262/Test262Harness.settings.json') }}
+        key: test262-generated-${{ hashFiles('Jint.Tests.Test262/Test262Harness.settings.json') }}-shard${{ matrix.shard }}-of-${{ env.TEST262_SHARD_COUNT }}
+        enableCrossOsArchive: true
+
+    - name: Generate shard
+      if: steps.cache.outputs.cache-hit != 'true'
+      env:
+        TEST262_SHARD_INDEX: ${{ matrix.shard }}
+      run: dotnet build Jint.Tests.Test262/Jint.Tests.Test262.csproj --configuration Release -t:GenerateTestSuite
+
+  # Validate Test262 on Linux, sharded.
+  test262:
+
+    needs: generate-test262
+
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [0, 1, 2, 3]
+
+    runs-on: ubuntu-latest
+
+    env:
+      TEST262_SHARD_INDEX: ${{ matrix.shard }}
+
+    steps:
+
+    - name: Setup NET
+      uses: actions/setup-dotnet@v5
+      with:
+        dotnet-version: 10.0
+
+    - name: Checkout source code
+      uses: actions/checkout@v7
+
+    - name: Restore Test262 generated shard
+      uses: actions/cache@v6
+      with:
+        path: Jint.Tests.Test262/Generated
+        key: test262-generated-${{ hashFiles('Jint.Tests.Test262/Test262Harness.settings.json') }}-shard${{ matrix.shard }}-of-${{ env.TEST262_SHARD_COUNT }}
+        enableCrossOsArchive: true
 
     - name: Test
-      run: dotnet test --configuration Release --logger "console;verbosity=quiet" --logger "GitHubActions;summary-include-passed=false;summary-include-skipped=false"
+      run: dotnet test Jint.Tests.Test262/Jint.Tests.Test262.csproj --configuration Release --logger "console;verbosity=quiet" --logger "GitHubActions;summary-include-passed=false;summary-include-skipped=false"
+
+  # Pack and publish the preview package once, after the suites pass.
+  publish:
+
+    needs: [ unit, test262 ]
+
+    runs-on: ubuntu-latest
+
+    steps:
+
+    - name: Setup NET
+      uses: actions/setup-dotnet@v5
+      with:
+        dotnet-version: 10.0
+
+    - name: Checkout source code
+      uses: actions/checkout@v7
 
     - name: Pack with dotnet
       run: dotnet pack Jint/Jint.csproj --output artifacts --configuration Release -p:VersionSuffix=preview-$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=True

.github/workflows/pr.yml

@@ -15,9 +15,10 @@ concurrency:
 
 jobs:
 
-  test:
+  # Everything except Test262: fast (< 1 min), so run the whole set once per OS.
+  unit:
 
-    name: ${{ matrix.os.name }}
+    name: unit - ${{ matrix.os.name }}
 
     strategy:
       fail-fast: false
@@ -38,14 +39,137 @@ jobs:
         dotnet-version: 10.0
 
     - name: Checkout source code
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
 
-    - name: Cache Test262 generated suite
-      uses: actions/cache@v5
+    - name: Test
+      shell: bash
+      run: |
+        set -e
+        for proj in \
+          Jint.Tests/Jint.Tests.csproj \
+          Jint.Tests.PublicInterface/Jint.Tests.PublicInterface.csproj \
+          Jint.Tests.CommonScripts/Jint.Tests.CommonScripts.csproj \
+          Jint.Tests.SourceGenerators/Jint.Tests.SourceGenerators.csproj; do
+          echo "::group::$proj"
+          dotnet test "$proj" --configuration Release \
+            --logger "console;verbosity=quiet" \
+            --logger "GitHubActions;summary-include-passed=false;summary-include-skipped=false"
+          echo "::endgroup::"
+        done
+
+  # Generate each Test262 shard once (codegen is OS-independent) and publish it to a cross-OS
+  # cache. Keyed per shard, so the previous same-key race between OS jobs is gone. These shards
+  # feed the sharded linux/windows test jobs below.
+  generate-test262:
+
+    name: generate test262 (shard ${{ matrix.shard }})
+
+    env:
+      TEST262_SHARD_COUNT: 4
+
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [0, 1, 2, 3]
+
+    runs-on: ubuntu-latest
+
+    steps:
+
+    - name: Setup NET
+      uses: actions/setup-dotnet@v5
+      with:
+        dotnet-version: 10.0
+
+    - name: Checkout source code
+      uses: actions/checkout@v7
+
+    - name: Cache Test262 generated shard
+      id: cache
+      uses: actions/cache@v6
       with:
         path: Jint.Tests.Test262/Generated
-        key: test262-generated-${{ hashFiles('Jint.Tests.Test262/Test262Harness.settings.json') }}
+        key: test262-generated-${{ hashFiles('Jint.Tests.Test262/Test262Harness.settings.json') }}-shard${{ matrix.shard }}-of-${{ env.TEST262_SHARD_COUNT }}
+        enableCrossOsArchive: true
+
+    - name: Generate shard
+      if: steps.cache.outputs.cache-hit != 'true'
+      env:
+        TEST262_SHARD_INDEX: ${{ matrix.shard }}
+      run: dotnet build Jint.Tests.Test262/Jint.Tests.Test262.csproj --configuration Release -t:GenerateTestSuite
+
+  # x64 linux and windows are the slow runners: shard Test262 4 ways across them. Each job restores
+  # its shard from the cross-OS cache, then compiles and runs only its ~1/4 slice of the suite.
+  test262-sharded:
+
+    name: test262 - ${{ matrix.os.name }} (shard ${{ matrix.shard }})
+
+    needs: generate-test262
+
+    env:
+      TEST262_SHARD_COUNT: 4
+      TEST262_SHARD_INDEX: ${{ matrix.shard }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - { name: 'linux', image: 'ubuntu-latest' }
+          - { name: 'windows', image: 'windows-latest' }
+        shard: [0, 1, 2, 3]
+
+    runs-on: ${{ matrix.os.image }}
+
+    steps:
+
+    - name: Setup NET
+      uses: actions/setup-dotnet@v5
+      with:
+        dotnet-version: 10.0
+
+    - name: Checkout source code
+      uses: actions/checkout@v7
+
+    - name: Restore Test262 generated shard
+      uses: actions/cache@v6
+      with:
+        path: Jint.Tests.Test262/Generated
+        key: test262-generated-${{ hashFiles('Jint.Tests.Test262/Test262Harness.settings.json') }}-shard${{ matrix.shard }}-of-${{ env.TEST262_SHARD_COUNT }}
+        enableCrossOsArchive: true
 
     - name: Test
-      run: dotnet test --configuration Release --logger "console;verbosity=quiet" --logger "GitHubActions;summary-include-passed=false;summary-include-skipped=false"
+      run: dotnet test Jint.Tests.Test262/Jint.Tests.Test262.csproj --configuration Release --logger "console;verbosity=quiet" --logger "GitHubActions;summary-include-passed=false;summary-include-skipped=false"
+
+  # macOS and ARM runners are scarcer (and already fast per-core), so run the full suite unsharded
+  # there — one runner each. The generated suite is cached per-OS (no cross-OS sharing needed).
+  test262-full:
+
+    name: test262 - ${{ matrix.os.name }}
 
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - { name: 'linux - ARM', image: 'ubuntu-24.04-arm' }
+          - { name: 'macos', image: 'macos-latest' }
+
+    runs-on: ${{ matrix.os.image }}
+
+    steps:
+
+    - name: Setup NET
+      uses: actions/setup-dotnet@v5
+      with:
+        dotnet-version: 10.0
+
+    - name: Checkout source code
+      uses: actions/checkout@v7
+
+    - name: Cache Test262 generated suite
+      uses: actions/cache@v6
+      with:
+        path: Jint.Tests.Test262/Generated
+        key: test262-generated-${{ hashFiles('Jint.Tests.Test262/Test262Harness.settings.json') }}-full
+
+    - name: Test
+      run: dotnet test Jint.Tests.Test262/Jint.Tests.Test262.csproj --configuration Release --logger "console;verbosity=quiet" --logger "GitHubActions;summary-include-passed=false;summary-include-skipped=false"

Jint.Tests.Test262/.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "test262harness.console": {
-      "version": "1.0.6",
+      "version": "1.1.0",
       "commands": [
         "test262"
       ]

Jint.Tests.Test262/Jint.Tests.Test262.csproj

@@ -45,12 +45,30 @@
       <_SettingsFilePath>$([System.IO.Path]::Combine($(MSBuildThisFileDirectory), $(SettingsFile)))</_SettingsFilePath>
       <_GeneratedDirPath>$([System.IO.Path]::Combine($(MSBuildThisFileDirectory), $(GeneratedTestSuiteDir)))</_GeneratedDirPath>
     </PropertyGroup>
-
+
+    <!--
+      Optional suite sharding. A CI matrix sets TEST262_SHARD_COUNT / TEST262_SHARD_INDEX (surfaced as
+      MSBuild properties) so each job only generates, compiles and runs its deterministic slice of the
+      suite. Defaults to a single shard = the full suite, so local builds are unaffected.
+    -->
+    <PropertyGroup>
+      <Test262ShardCount Condition="'$(Test262ShardCount)' == ''">$(TEST262_SHARD_COUNT)</Test262ShardCount>
+      <Test262ShardCount Condition="'$(Test262ShardCount)' == ''">1</Test262ShardCount>
+      <Test262ShardIndex Condition="'$(Test262ShardIndex)' == ''">$(TEST262_SHARD_INDEX)</Test262ShardIndex>
+      <Test262ShardIndex Condition="'$(Test262ShardIndex)' == ''">0</Test262ShardIndex>
+      <_Test262ShardArgs Condition="$(Test262ShardCount) &gt; 1">--shard-count $(Test262ShardCount) --shard-index $(Test262ShardIndex)</_Test262ShardArgs>
+    </PropertyGroup>
+
     <!-- Compute current settings file hash -->
     <GetFileHash Files="$(_SettingsFilePath)" Algorithm="SHA256">
       <Output TaskParameter="Hash" PropertyName="_CurrentSettingsHash" />
     </GetFileHash>
-
+
+    <!-- Fold the shard identity into the hash so switching shards in place forces a regeneration -->
+    <PropertyGroup>
+      <_CurrentSettingsHash>$(_CurrentSettingsHash)-shard$(Test262ShardIndex)-of-$(Test262ShardCount)</_CurrentSettingsHash>
+    </PropertyGroup>
+
     <!-- Read stored hash if it exists -->
     <ReadLinesFromFile File="$(_SettingsHashFilePath)" Condition="Exists('$(_SettingsHashFilePath)')">
       <Output TaskParameter="Lines" PropertyName="_StoredSettingsHash" />
@@ -71,7 +89,7 @@
 
     <!-- Run generation if needed -->
     <Exec Command="dotnet tool restore" Condition="'$(_NeedsRegeneration)' == 'true'" />
-    <Exec Command="dotnet test262 generate" Condition="'$(_NeedsRegeneration)' == 'true'" />
+    <Exec Command="dotnet test262 generate $(_Test262ShardArgs)" Condition="'$(_NeedsRegeneration)' == 'true'" />
 
     <!-- Store the new hash after successful generation -->
     <WriteLinesToFile File="$(_SettingsHashFilePath)" Lines="$(_CurrentSettingsHash)" Overwrite="true" Condition="'$(_NeedsRegeneration)' == 'true'" />