diff --git a/.github/workflows/fuzz.yaml b/.github/workflows/fuzz.yaml new file mode 100644 index 0000000..1529113 --- /dev/null +++ b/.github/workflows/fuzz.yaml @@ -0,0 +1,105 @@ +name: Continuous fuzz (FsCheck) + +# Long-run property-based fuzzing per issue #170. Executes only the +# tests tagged `[Trait("Category", "Fuzz")]` in +# tests/Wolfgang.TryPattern.Tests.Unit/FuzzTests.cs — 100,000 +# randomized cases per property, ~2–5 min wall-clock on ubuntu. +# +# On any failure, opens a `kind:fuzz-finding` issue with the failing +# seed + counter-example pulled from the test output so a maintainer +# can reproduce with the exact input FsCheck shrunk to. + +on: + schedule: + - cron: '0 4 * * 1' # Mondays 04:00 UTC + workflow_dispatch: + +permissions: + contents: read + issues: write # File a kind:fuzz-finding issue on failure. + +concurrency: + group: fuzz + cancel-in-progress: false + +jobs: + fuzz: + name: Long-run FsCheck fuzz + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Setup .NET + uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5 + with: + dotnet-version: '10.0.x' + + - name: Run fuzz-tagged properties + id: fuzz + # `--filter "Category=Fuzz"` selects only the FuzzTests class + # (marked with [Trait("Category", "Fuzz")]). Modern TFM only — + # net10.0 — since the fuzz semantics don't vary by TFM and + # running the matrix would just multiply wall-clock. + run: | + set -o pipefail + mkdir -p ./fuzz-output + dotnet test tests/Wolfgang.TryPattern.Tests.Unit \ + -c Release \ + -f net10.0 \ + --filter "Category=Fuzz" \ + --logger "console;verbosity=detailed" \ + 2>&1 | tee ./fuzz-output/log.txt + + - name: Extract failing seeds + if: failure() && steps.fuzz.outcome == 'failure' + # FsCheck prints "Falsifiable, after N tests" + "Original:" + + # "Shrunk:" lines on failure. Grep those out so the auto-issue + # can carry a compact reproducer. + run: | + { + echo '## Failing seeds and shrunk counter-examples' + echo + echo 'Extracted from `fuzz-output/log.txt`:' + echo + echo '```' + grep -E 'Falsifiable|Original:|Shrunk:|with seed' ./fuzz-output/log.txt || echo '(no matches — check full log artifact)' + echo '```' + } > ./fuzz-output/summary.md + cat ./fuzz-output/summary.md + + - name: Upload fuzz output + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fuzz-output-${{ github.run_id }} + path: ./fuzz-output/ + retention-days: 60 + + - name: File fuzz-finding issue on failure + if: failure() && steps.fuzz.outcome == 'failure' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + summary=$(cat ./fuzz-output/summary.md 2>/dev/null || echo "See workflow log.") + gh issue create \ + --repo "$REPO" \ + --title "Fuzz finding on $(date -u +%Y-%m-%d): FsCheck property refuted" \ + --label kind:fuzz-finding \ + --body "$(cat < + ( + x => + { + int sink = 0; + Result r = Try.Run(() => { sink = x; }); + return r.Succeeded && sink == x; + } + ); + } + + + [Property(MaxTest = LongRunMaxTest)] + public Property Fuzz_Try_Run_of_throwing_action_carries_message(NonEmptyString message) + { + if (message is null) throw new ArgumentNullException(nameof(message)); + string msg = message.Get; + // Exclude whitespace-only strings — Result.Failure requires + // non-whitespace and a whitespace exception message causes + // Try.Run itself to throw (tracked as bug #273). Skip via a + // trivial-true property so FsCheck's generator keeps producing + // new inputs, but this specific case doesn't count as a + // failure until #273 is fixed. + if (string.IsNullOrWhiteSpace(msg)) + { + return true.ToProperty(); + } + Result r = Try.Run(() => throw new InvalidOperationException(msg)); + return (r.Failed && string.Equals(r.ErrorMessage, msg, StringComparison.Ordinal)).ToProperty(); + } + + + [Property(MaxTest = LongRunMaxTest)] + public Property Fuzz_Try_Run_generic_returns_input_value(int value) + { + Result r = Try.Run(() => value); + return (r.Succeeded && r.Value == value).ToProperty(); + } + + + [Property(MaxTest = LongRunMaxTest)] + public Property Fuzz_AllSucceeded_is_negation_of_AnyFailed(bool[] successFlags) + { + if (successFlags is null || successFlags.Length == 0) + { + return true.ToProperty(); + } + + Result[] results = new Result[successFlags.Length]; + for (int i = 0; i < successFlags.Length; i++) + { + results[i] = successFlags[i] + ? Result.Success() + : Result.Failure($"f{i}"); + } + + bool allOk = Result.AllSucceeded(results); + bool anyFail = Result.AnyFailed(results); + return (allOk == !anyFail).ToProperty(); + } +} diff --git a/tests/Wolfgang.TryPattern.Tests.Unit/PropertyTests.cs b/tests/Wolfgang.TryPattern.Tests.Unit/PropertyTests.cs new file mode 100644 index 0000000..e701fcf --- /dev/null +++ b/tests/Wolfgang.TryPattern.Tests.Unit/PropertyTests.cs @@ -0,0 +1,141 @@ +// Property-based fuzz tests per issue #170. +// +// FsCheck generates random inputs across each theorem's parameter +// space and checks the property holds for every generated case. +// Each [Property] runs 100 randomized inputs by default; a +// counter-example (if any) is minimized and reported in the test +// output. +// +// Try-Pattern's specification is small but has real properties worth +// enforcing: +// +// - Try.Run(non-throwing action) → always Succeeded +// - Try.Run(throwing action) → always Failed with round-tripped message +// - Try.Run(fn returning value) → Succeeded with .Value == value +// - Result.AllSucceeded ⇔ !Result.AnyFailed (on non-empty inputs) +// - Result.Flatten idempotence: Flatten with all-successes is Success +// +// A property failure means the specification described in the class- +// level comment is violated — investigate before dismissing. + +using System; +using FsCheck; +using FsCheck.Xunit; +using Xunit; + +namespace Wolfgang.TryPattern.Tests.Unit; + +public class PropertyTests +{ + [Property] + public Property Try_Run_of_non_throwing_action_always_succeeds() + { + return Prop.ForAll + ( + x => + { + int sink = 0; + Result r = Try.Run(() => { sink = x; }); + return r.Succeeded && sink == x; + } + ); + } + + + [Property] + public Property Try_Run_of_throwing_action_carries_message_round_trip(NonEmptyString message) + { + if (message is null) throw new ArgumentNullException(nameof(message)); + // NonEmptyString excludes only the empty string; Result.Failure + // additionally rejects whitespace-only strings and there's a + // latent bug (#273) that lets a whitespace exception message + // escape from Try.Run. Skip that case so the fast-path + // property is well-formed vs the current API. + string msg = message.Get; + if (string.IsNullOrWhiteSpace(msg)) + { + return true.ToProperty(); + } + Result r = Try.Run(() => throw new InvalidOperationException(msg)); + return (r.Failed && string.Equals(r.ErrorMessage, msg, StringComparison.Ordinal)).ToProperty(); + } + + + [Property] + public Property Try_Run_generic_returns_input_value(int value) + { + Result r = Try.Run(() => value); + return (r.Succeeded && r.Value == value).ToProperty(); + } + + + [Property] + public Property AllSucceeded_is_negation_of_AnyFailed(bool[] successFlags) + { + // FsCheck may generate the empty array. The API's semantics + // on empty input are edge cases (both return true / false + // depending on interpretation). Skip empty via a precondition. + if (successFlags is null || successFlags.Length == 0) + { + return true.ToProperty(); + } + + Result[] results = new Result[successFlags.Length]; + for (int i = 0; i < successFlags.Length; i++) + { + results[i] = successFlags[i] + ? Result.Success() + : Result.Failure($"f{i}"); + } + + bool allOk = Result.AllSucceeded(results); + bool anyFail = Result.AnyFailed(results); + return (allOk == !anyFail).ToProperty(); + } + + + [Property] + public Property Flatten_of_all_successes_is_success(NonNegativeInt count) + { + if (count is null) throw new ArgumentNullException(nameof(count)); + // Bound the count so FsCheck doesn't allocate a huge array + // just because it generated int.MaxValue. + int n = Math.Min(count.Get, 64); + Result[] results = new Result[n]; + for (int i = 0; i < n; i++) + { + results[i] = Result.Success(); + } + + Result flat = Result.Flatten(results); + return flat.Succeeded.ToProperty(); + } + + + [Property] + public Property Flatten_with_at_least_one_failure_is_failure(int[] failIndices) + { + // At least one failure index → the flattened Result is Failed. + // FsCheck can generate empty or all-positive arrays; use an + // upfront normalization to ensure at least one failure. + if (failIndices is null || failIndices.Length == 0) + { + return true.ToProperty(); + } + + int n = failIndices.Length; + Result[] results = new Result[n]; + for (int i = 0; i < n; i++) + { + results[i] = failIndices[i] % 2 == 0 + ? Result.Failure($"f{i}") + : Result.Success(); + } + + // Guarantee at least one failure regardless of input. + results[0] = Result.Failure("guaranteed"); + + Result flat = Result.Flatten(results); + return flat.Failed.ToProperty(); + } +} diff --git a/tests/Wolfgang.TryPattern.Tests.Unit/Wolfgang.TryPattern.Tests.Unit.csproj b/tests/Wolfgang.TryPattern.Tests.Unit/Wolfgang.TryPattern.Tests.Unit.csproj index 86acfe4..b40e8ba 100644 --- a/tests/Wolfgang.TryPattern.Tests.Unit/Wolfgang.TryPattern.Tests.Unit.csproj +++ b/tests/Wolfgang.TryPattern.Tests.Unit/Wolfgang.TryPattern.Tests.Unit.csproj @@ -39,6 +39,10 @@ + +