diff --git a/.github/workflows/stryker.yaml b/.github/workflows/stryker.yaml index 930f7d64..dc5508fa 100644 --- a/.github/workflows/stryker.yaml +++ b/.github/workflows/stryker.yaml @@ -1,15 +1,30 @@ -# Stryker mutation testing +# Stryker mutation testing (#205 — release-gate quality bar) # -# Runs the Stryker.NET mutation tester against the repo's test projects to -# measure mutation score. Mutation runs are slow — triggered manually -# (workflow_dispatch) and on a weekly schedule, not on every PR. +# Runs the Stryker.NET mutation tester to measure mutation score. The +# stryker-config.json `break` threshold is the documented floor: `dotnet stryker` +# exits non-zero (failing the job) if the score drops below it, so this is an +# enforced gate, not just a report. Ratchet the floor UP over time; never down. +# +# Triggers: +# - pull_request touching src/** — gates a PR that would drop the score below +# the floor, BEFORE merge. Mutation runs are slow (tens of minutes), so this +# is scoped to source changes only; docs/test-only/workflow PRs skip it. +# - schedule (weekly) + workflow_dispatch — full-project trend / on-demand runs. # # The workflow looks for a stryker-config.json at the repo root or under -# tests/**/. If none is present the run is a no-op (Stryker setup is a -# per-repo follow-up; this file is the canonical infrastructure). +# tests/**/. If none is present the run is a no-op. name: Stryker (mutation testing) on: + pull_request: + branches: + - main + - vNext + paths: + - 'src/**/*.cs' + - 'tests/**/*.cs' + - 'stryker-config.json' + - '.github/workflows/stryker.yaml' workflow_dispatch: schedule: - cron: '0 6 * * 0' # weekly Sunday 06:00 UTC @@ -22,6 +37,9 @@ jobs: name: Run Stryker runs-on: ubuntu-latest timeout-minutes: 60 + permissions: + contents: write # publish the score-history chart to gh-pages (non-PR runs only) + issues: write # upsert the single survivors tracking issue (non-PR runs only) steps: - name: Check out repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -65,13 +83,18 @@ jobs: - name: Install dotnet-stryker if: steps.check.outputs.found == 'true' - run: dotnet tool update -g dotnet-stryker || dotnet tool install -g dotnet-stryker + # Pinned: mutation score / the gate must be reproducible, and a floating + # Stryker can change instrumentation or break on an analyzer/Roslyn bump. + run: dotnet tool update -g dotnet-stryker --version 4.16.0 || dotnet tool install -g dotnet-stryker --version 4.16.0 - name: Run Stryker if: steps.check.outputs.found == 'true' shell: bash run: | - set -e + # pipefail so the gate is preserved: `dotnet stryker` exits non-zero when + # the score is below the config's `break` floor, and `| tee` must NOT mask + # that. The tee'd log is parsed by the next step for the score. + set -eo pipefail shopt -s globstar nullglob # Run BOTH a root stryker-config.json (if present) AND any # tests/**/stryker-config.json suites. The Detect step collects @@ -80,14 +103,14 @@ jobs: ran=0 if [ -f stryker-config.json ]; then echo "::group::Stryker with root stryker-config.json" - dotnet stryker --config-file stryker-config.json + dotnet stryker --config-file stryker-config.json 2>&1 | tee -a "$GITHUB_WORKSPACE/stryker.log" echo "::endgroup::" ran=1 fi for cfg in tests/**/stryker-config.json; do dir=$(dirname "$cfg") echo "::group::Stryker in $dir" - (cd "$dir" && dotnet stryker) + (cd "$dir" && dotnet stryker 2>&1 | tee -a "$GITHUB_WORKSPACE/stryker.log") echo "::endgroup::" ran=1 done @@ -96,6 +119,88 @@ jobs: exit 1 fi + - name: Extract mutation score + id: score + # always(): publish/track even when the gate FAILED (a low score is exactly + # when the trend + survivor list matter most). + if: always() && steps.check.outputs.found == 'true' + shell: bash + run: | + # Stryker prints e.g. "The final mutation score is 74.40 %". + score=$(grep -oE 'final mutation score is [0-9]+(\.[0-9]+)?' "$GITHUB_WORKSPACE/stryker.log" 2>/dev/null \ + | grep -oE '[0-9]+(\.[0-9]+)?' | tail -1) + if [ -z "$score" ]; then + echo "::warning::Could not parse a mutation score from the Stryker output." + score=0 + fi + echo "score=$score" >> "$GITHUB_OUTPUT" + printf '[{"name":"Mutation score","unit":"%%","value":%s}]\n' "$score" > mutation-score.json + echo "Mutation score: ${score} %" + + - name: Publish mutation-score history to gh-pages + # Trend chart under gh-pages /dev/stryker (same action + pin as the BDN + # benchmark charts). Non-PR only — one data point per scheduled/manual run. + if: always() && steps.check.outputs.found == 'true' && github.event_name != 'pull_request' + uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1 + with: + name: Mutation score + tool: 'customBiggerIsBetter' + output-file-path: mutation-score.json + gh-pages-branch: gh-pages + benchmark-data-dir-path: dev/stryker + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + summary-always: true + # The `break` floor is the enforced gate; this chart is trend visibility, + # so it must not double-fail the run. + fail-on-alert: false + + - name: Track surviving mutants + # Writes the survivor worklist to the run summary and keeps ONE rolling + # kind:mutation-survives issue up to date (deliberately not one issue per + # survivor — there are dozens). Best-effort: never fails the run. + if: always() && steps.check.outputs.found == 'true' && github.event_name != 'pull_request' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + set -uo pipefail + shopt -s globstar nullglob + reports=(**/StrykerOutput/**/reports/mutation-report.json) + if (( ${#reports[@]} == 0 )); then + echo "::warning::No mutation-report.json found; skipping survivor tracking." + exit 0 + fi + report="${reports[-1]}" + survivors=$(jq -r '.files | to_entries[] | .key as $f | .value.mutants[]? + | select(.status=="Survived") + | "- `\($f):\(.location.start.line)` — \(.mutatorName)"' "$report" | sort) + count=$(printf '%s\n' "$survivors" | grep -c '^- ' || true) + score='${{ steps.score.outputs.score }}' + + { + echo "## Surviving mutants: ${count}" + echo "" + echo "Mutation score **${score} %** (floor 70). Kill these to raise it:" + echo "" + printf '%s\n' "$survivors" + } >> "$GITHUB_STEP_SUMMARY" + + # Ensure the label exists, then upsert a single rolling tracking issue. + gh label create "kind:mutation-survives" --repo "$GITHUB_REPOSITORY" \ + --color BFD4F2 --description "A mutant survived — a test gap to close" 2>/dev/null || true + title="Mutation survivors (${count}) — kill these to raise the score" + body=$(printf '_Auto-updated by the Stryker workflow (run %s). Mutation score **%s %%** (floor 70)._\n\n%s\n' \ + "$GITHUB_RUN_ID" "$score" "$survivors") + existing=$(gh issue list --repo "$GITHUB_REPOSITORY" --label "kind:mutation-survives" \ + --state open --json number --jq '.[0].number' 2>/dev/null || true) + if [ -n "$existing" ]; then + gh issue edit "$existing" --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body" || true + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body" \ + --label "kind:mutation-survives" || true + fi + - name: Upload Stryker report if: always() && steps.check.outputs.found == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/docs/mutation-testing.md b/docs/mutation-testing.md new file mode 100644 index 00000000..6653c3ed --- /dev/null +++ b/docs/mutation-testing.md @@ -0,0 +1,60 @@ +# Mutation testing (#205) + +Mutation testing seeds deliberate faults ("mutants") into the source and checks +that the test suite catches them. A *survived* mutant is a change to behaviour +that **no test noticed** — a hole in the suite that line coverage can't see. + +This repo runs [Stryker.NET](https://stryker-mutator.io/docs/stryker-net/introduction/) +as an **enforced release-gate quality bar**, not just a report. + +## The floor + +`stryker-config.json` sets `thresholds.break`, which makes `dotnet stryker` exit +non-zero (failing the job) when the mutation score drops below it. + +| | Value | +|---|---| +| Baseline score (full project, 2026-07-22, Stryker 4.16.0) | **74.4 %** — 248 killed / 83 survived / 2 timeout of 333 tested | +| **Enforced floor (`break`)** | **70 %** | + +The floor sits a few points below the measured baseline so normal CI-runner +variance (a slow runner can turn a killed mutant into a timeout, nudging the +score) doesn't cause a spurious failure, while a real regression — deleting a +test, or adding untested behaviour — trips it. + +**Policy: ratchet the floor UP, never down.** As survivors are killed and the +score climbs, raise `break` to lock in the gain. Lowering it to make a red build +pass defeats the point — fix the test gap instead. + +## How it runs + +- **Pull requests that touch `src/**`** run the gate before merge + (`.github/workflows/stryker.yaml`). It is scoped to source changes because a + full run takes tens of minutes; docs/test-only/workflow PRs skip it. +- **Weekly schedule + `workflow_dispatch`** run it on demand / for the trend. + +## Running locally + +```bash +dotnet tool install --global dotnet-stryker --version 4.16.0 +dotnet stryker # full project (~4 min) +dotnet stryker --mutate "**/Report.cs" # a single file, faster +``` + +The HTML report under `StrykerOutput/**/reports/` lists every survived mutant with +its file, line, and the mutation applied — the worklist for raising the score. + +## Score history & survivor tracking + +On the weekly schedule (and manual `workflow_dispatch`) runs — not on PRs — the +workflow also: + +- **Charts the score trend** on the `gh-pages` branch under `/dev/stryker/`, using + the same `github-action-benchmark` action as the BDN benchmark charts + (`customBiggerIsBetter`, one data point per run). This is trend visibility only — + it never fails the run; the `break` floor is the gate. +- **Tracks survivors** by writing the survivor worklist (file:line + mutator) to the + run's job summary and keeping **one** rolling `kind:mutation-survives` issue up to + date. It is deliberately a single rolling issue, not one issue per survivor — + there are dozens, and per-mutant issues would be noise. Work the list down from + the issue (or the uploaded HTML report), then raise the floor. diff --git a/stryker-config.json b/stryker-config.json index fff2804e..2275171b 100644 --- a/stryker-config.json +++ b/stryker-config.json @@ -13,7 +13,7 @@ "thresholds": { "high": 90, "low": 75, - "break": 0 + "break": 70 } } }