test: concurrency / race-condition stress testing with Coyote (#207)

.github/workflows/coyote.yaml

@@ -0,0 +1,87 @@
+name: Concurrency (Coyote)
+
+# Systematic concurrency / race-condition testing (#207) with Microsoft Coyote.
+# Coyote rewrites the assemblies to control the task scheduler, then replays each
+# test under thousands of distinct interleavings — surfacing races/deadlocks that a
+# normal single-schedule run never hits. `coyote test` exits non-zero on a found
+# bug, so this is an enforced gate.
+#
+# The Coyote project targets net8.0: the Coyote 1.7.x CLI cannot load net10.0
+# assemblies (it predates that runtime), and the concurrency being explored is
+# runtime-agnostic. Triggers:
+#   - pull_request touching src/** or the concurrency project — a lighter run
+#     (1000 iterations/test) gates races before merge.
+#   - schedule (weekly) + workflow_dispatch — the deep run (10000 iterations/test).
+#
+# NOT implemented: the optional 24h soak (handle/thread/memory-leak hunting via
+# dotnet-counters). It requires a persistent self-hosted runner, which this project
+# does not have; add it here behind a `runs-on: [self-hosted]` job when one exists.
+
+on:
+  pull_request:
+    branches:
+      - main
+      - vNext
+    paths:
+      - 'src/**/*.cs'
+      - 'tests/Wolfgang.Etl.Abstractions.Tests.Concurrency/**'
+      - '.github/workflows/coyote.yaml'
+  schedule:
+    - cron: '0 5 * * 1' # weekly Monday 05:00 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  coyote:
+    name: Coyote systematic exploration
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      PROJECT: tests/Wolfgang.Etl.Abstractions.Tests.Concurrency
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6
+        with:
+          dotnet-version: |
+            8.0.x
+            10.0.x
+
+      - name: Build the concurrency project (Release)
+        run: dotnet build "$PROJECT/Wolfgang.Etl.Abstractions.Tests.Concurrency.csproj" -c Release
+
+      - name: Install Coyote CLI
+        run: |
+          dotnet tool install --global Microsoft.Coyote.CLI --version 1.7.11
+          echo "$HOME/.dotnet/tools" >> "$GITHUB_PATH"
+
+      - name: Rewrite assemblies for the controlled scheduler
+        working-directory: ${{ env.PROJECT }}
+        run: coyote rewrite coyote.json
+
+      - name: Run Coyote systematic tests
+        working-directory: ${{ env.PROJECT }}
+        run: |
+          set -euo pipefail
+          # Deep run on schedule/dispatch; a lighter gate on PRs to keep them fast.
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            iterations=1000
+          else
+            iterations=10000
+          fi
+          dll="bin/Release/net8.0/Wolfgang.Etl.Abstractions.Tests.Concurrency.dll"
+          methods=(
+            Concurrent_item_count_increments_never_lose_an_update
+            Dispose_racing_enumeration_never_deadlocks
+          )
+          for m in "${methods[@]}"; do
+            echo "::group::coyote test $m ($iterations iterations)"
+            coyote test "$dll" --method "$m" --iterations "$iterations"
+            echo "::endgroup::"
+          done

tests/Wolfgang.Etl.Abstractions.Tests.Concurrency/ConcurrencyTests.cs

@@ -0,0 +1,110 @@
+using System.Runtime.CompilerServices;
+using Microsoft.Coyote.Specifications;
+using Microsoft.Coyote.SystematicTesting;
+
+namespace Wolfgang.Etl.Abstractions.Tests.Concurrency;
+
+/// <summary>
+/// Concurrency / race-condition stress tests (#207) driven by Microsoft Coyote's
+/// systematic scheduler. Each <c>[Test]</c> method is replayed under thousands of
+/// distinct task interleavings by <c>coyote test</c> (see
+/// <c>.github/workflows/coyote.yaml</c>); a race, deadlock, or assertion violation
+/// on any explored schedule fails the run.
+/// </summary>
+public static class ConcurrencyTests
+{
+    /// <summary>
+    /// Two tasks concurrently drive the base class's per-item counter. Under every
+    /// interleaving Coyote explores, no increment may be lost — the counter is an
+    /// <see cref="System.Threading.Interlocked"/> operation. A non-atomic <c>++</c>
+    /// would surface here as a lost update on some schedule.
+    /// </summary>
+    [Microsoft.Coyote.SystematicTesting.Test]
+    public static async Task Concurrent_item_count_increments_never_lose_an_update()
+    {
+        var harness = new CounterHarness();
+        const int perTask = 5;
+
+        var first = Task.Run(() =>
+        {
+            for (var i = 0; i < perTask; i++)
+            {
+                harness.Bump();
+            }
+        });
+
+        var second = Task.Run(() =>
+        {
+            for (var i = 0; i < perTask; i++)
+            {
+                harness.Bump();
+            }
+        });
+
+        await Task.WhenAll(first, second);
+
+        Specification.Assert(
+            harness.Count == 2 * perTask,
+            $"Lost update under a concurrent schedule: expected {2 * perTask}, got {harness.Count}.");
+    }
+
+
+    /// <summary>
+    /// Races <see cref="System.IAsyncDisposable.DisposeAsync"/> against an in-flight
+    /// enumeration of the same extractor. Under every interleaving the run must
+    /// terminate — no deadlock, and no unobserved exception other than the expected
+    /// cancellation/disposal signals.
+    /// </summary>
+    [Microsoft.Coyote.SystematicTesting.Test]
+    public static async Task Dispose_racing_enumeration_never_deadlocks()
+    {
+        var extractor = new CounterHarness();
+
+        var consume = Task.Run(async () =>
+        {
+            try
+            {
+                await foreach (var _ in extractor.ExtractAsync())
+                {
+                    // drain
+                }
+            }
+            catch (ObjectDisposedException)
+            {
+                // acceptable outcome of racing disposal
+            }
+            catch (OperationCanceledException)
+            {
+                // acceptable outcome of racing disposal
+            }
+        });
+
+        var dispose = Task.Run(async () => await extractor.DisposeAsync());
+
+        await Task.WhenAll(consume, dispose);
+        // Reaching here means neither task deadlocked on any explored schedule.
+    }
+
+
+    // Surfaces the protected per-item counter for the increment race, and provides a
+    // trivial async source for the dispose race.
+    private sealed class CounterHarness : ExtractorBase<int, Report>
+    {
+        public void Bump() => IncrementCurrentItemCount();
+
+        public int Count => CurrentItemCount;
+
+        protected override async IAsyncEnumerable<int> ExtractWorkerAsync(
+            [EnumeratorCancellation] CancellationToken token)
+        {
+            for (var i = 0; i < 4; i++)
+            {
+                token.ThrowIfCancellationRequested();
+                yield return i;
+                await Task.Yield();
+            }
+        }
+
+        protected override Report CreateProgressReport() => new(CurrentItemCount);
+    }
+}

tests/Wolfgang.Etl.Abstractions.Tests.Concurrency/Wolfgang.Etl.Abstractions.Tests.Concurrency.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <!-- Concurrency / race-condition stress testing with Microsoft Coyote (#207).
+       Coyote is a systematic-concurrency explorer: it rewrites the assemblies to
+       take control of the task scheduler, then replays each test method under
+       thousands of distinct interleavings to surface races/deadlocks a normal
+       run never hits. This project is driven by the `coyote` CLI from
+       .github/workflows/coyote.yaml — NOT by the xunit test job — so it is not a
+       standard test project. -->
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <LangVersion>latest</LangVersion>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <IsPackable>false</IsPackable>
+    <OutputType>Library</OutputType>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Coyote" Version="1.7.11" />
+    <PackageReference Include="Microsoft.Coyote.Test" Version="1.7.11" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Wolfgang.Etl.Abstractions\Wolfgang.Etl.Abstractions.csproj" />
+  </ItemGroup>
+
+</Project>

tests/Wolfgang.Etl.Abstractions.Tests.Concurrency/coyote.json

@@ -0,0 +1,7 @@
+{
+  "AssembliesPath": "bin/Release/net8.0",
+  "Assemblies": [
+    "Wolfgang.Etl.Abstractions.Tests.Concurrency.dll",
+    "Wolfgang.Etl.Abstractions.dll"
+  ]
+}