ci: add Stryker mutation-score enforcement + trend (#124)

.github/workflows/actions-audit.yaml

@@ -48,6 +48,11 @@ jobs:
         run: go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.7
 
       - name: Run actionlint
+        env:
+          # Gate on shellcheck warning+ only. The info/style nits (SC2012 "use
+          # find not ls", SC2035 "use ./*glob*") in the canonical pr.yaml are
+          # not worth failing every PR over; warnings and errors still gate.
+          SHELLCHECK_OPTS: --severity=warning
         run: actionlint -color
 
   zizmor:
@@ -76,9 +81,9 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         # Report-only for now: don't fail the job on findings, upload them to
         # Code Scanning instead. Remove `|| true` to turn this into a gate.
-        run: zizmor --format sarif .github/workflows/ > zizmor.sarif || true
+        run: zizmor --config .zizmor.yml --format sarif .github/workflows/ > zizmor.sarif || true
 
       - name: Upload zizmor SARIF
-        uses: github/codeql-action/upload-sarif@e0647621c2984b5ed2f768cb892365bf2a616ad1 # v4
+        uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4
         with:
           sarif_file: zizmor.sarif

.github/workflows/codeql.yaml

@@ -77,7 +77,7 @@ jobs:
 
     - name: Initialize CodeQL
       if: steps.check-csharp.outputs.has-csharp == 'true'
-      uses: github/codeql-action/init@e0647621c2984b5ed2f768cb892365bf2a616ad1 # v4
+      uses: github/codeql-action/init@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4
       with:
         languages: ${{ matrix.language }}
         # security-extended adds the broader security query pack on top of the
@@ -159,7 +159,7 @@ jobs:
     - name: Perform CodeQL Analysis
       id: perform-codeql-analysis
       if: steps.check-csharp.outputs.has-csharp == 'true'
-      uses: github/codeql-action/analyze@e0647621c2984b5ed2f768cb892365bf2a616ad1 # v4
+      uses: github/codeql-action/analyze@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4
       with:
         category: "/language:${{matrix.language}}"
 

.github/workflows/pr.yaml

@@ -275,7 +275,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v7
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         with:
           ref: refs/pull/${{ github.event.pull_request.number }}/head
           persist-credentials: false
@@ -337,7 +337,7 @@ jobs:
           echo "✅ Configuration files secured - using versions from main branch"
 
       - name: Setup .NET
-        uses: actions/setup-dotnet@v5
+        uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6
         with:
           # Match the test stages' SDK list so the solution's older-TFM projects
           # (test projects target netcoreapp3.1 / net5.0-net7.0) restore and build
@@ -394,7 +394,7 @@ jobs:
             --no-build
 
       - name: Upload SARIF to Code Scanning
-        uses: github/codeql-action/upload-sarif@v4
+        uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4
         with:
           sarif_file: inspect.sarif
 

.github/workflows/scorecard.yaml

@@ -45,6 +45,6 @@ jobs:
           publish_results: true
 
       - name: Upload SARIF to Code Scanning
-        uses: github/codeql-action/upload-sarif@e0647621c2984b5ed2f768cb892365bf2a616ad1 # v4
+        uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4
         with:
           sarif_file: results.sarif

.github/workflows/semgrep.yaml

@@ -57,6 +57,6 @@ jobs:
             --error || true
 
       - name: Upload Semgrep SARIF
-        uses: github/codeql-action/upload-sarif@e0647621c2984b5ed2f768cb892365bf2a616ad1 # v4
+        uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4
         with:
           sarif_file: semgrep.sarif

.github/workflows/stryker.yaml

@@ -9,6 +9,11 @@
 # - workflow_dispatch: manual ad-hoc runs (e.g. while iterating on tests)
 # - schedule (weekly Sunday 06:00 UTC): catch quality regressions between
 #   releases without burning CI time on every PR (mutation testing is slow)
+# - pull_request (src/** or tests/** changes): the #124 enforcement gate. The
+#   per-project stryker-config.json pins target-framework:net10.0, so a PR runs
+#   a single-TFM mutation pass (~minutes) rather than the full matrix, and the
+#   config's `break` threshold fails the PR if the score regresses below floor.
+# - push to main: re-measure and publish the score to the gh-pages trend.
 #
 # Stryker discovers each test project's mutation targets via the standard
 # project-reference graph. Two configuration modes are supported:
@@ -23,6 +28,17 @@ on:
   workflow_dispatch:
   schedule:
     - cron: '0 6 * * 0' # weekly Sunday 06:00 UTC
+  pull_request:
+    paths:
+      - 'src/**'
+      - 'tests/**'
+      - '.github/workflows/stryker.yaml'
+      - 'scripts/stryker-score.py'
+  push:
+    branches: [main]
+    paths:
+      - 'src/**'
+      - 'tests/**'
 
 permissions:
   contents: read
@@ -102,8 +118,20 @@ jobs:
             $tests | ForEach-Object { Write-Host "  $($_.FullName)" }
             Write-Host ""
 
+            # Opt-in gating (#124): only run test projects that ship a
+            # stryker-config.json. A project without one has no documented
+            # score floor yet, so running it with Stryker defaults would be an
+            # ungated, all-TFM pass that is slow on every PR and can't fail the
+            # gate meaningfully. Add a stryker-config.json next to a test
+            # project to bring it under the gate.
+            $ran = 0
             foreach ($proj in $tests) {
               $dir = $proj.DirectoryName
+              if (-not (Test-Path (Join-Path $dir 'stryker-config.json'))) {
+                Write-Host "::notice::No stryker-config.json in $dir - skipping (add one to enable the mutation gate for this project)."
+                continue
+              }
+              $ran++
               Write-Host "::group::Stryker in $dir"
               Push-Location $dir
               try {
@@ -114,6 +142,11 @@ jobs:
               }
               Write-Host "::endgroup::"
             }
+
+            if ($ran -eq 0) {
+              Write-Host "::notice::No test project has a stryker-config.json - nothing to gate."
+              exit 0
+            }
           }
 
           if ($failed -gt 0) {
@@ -130,3 +163,51 @@ jobs:
             **/StrykerOutput/**
           if-no-files-found: ignore
           retention-days: 30
+
+  # Publish the mutation score to the gh-pages trend (#124), charted by the same
+  # benchmark-action pattern the BDN benchmark trend uses. Separate job so only
+  # it carries `contents: write`; the gate job above stays least-privilege. Runs
+  # only on trusted, non-PR events (a fork PR must never push to gh-pages) and
+  # only when the gate above passed (needs: stryker).
+  publish-trend:
+    name: Publish mutation-score trend
+    needs: stryker
+    if: github.event_name == 'push' || github.event_name == 'schedule'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write   # push the score history to the gh-pages branch
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: '3.x'
+
+      - name: Download Stryker report
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: stryker-report-${{ github.run_id }}
+          path: stryker-report
+
+      - name: Extract mutation score
+        run: python scripts/stryker-score.py stryker-report stryker-score.json
+
+      - name: Publish score to gh-pages trend
+        uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1
+        with:
+          name: Mutation score
+          tool: 'customBiggerIsBetter'
+          output-file-path: stryker-score.json
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: dev/stryker
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Bigger-is-better: alert if the score drops. Report-only (the PR gate
+          # above is the hard enforcement); this line just annotates the trend.
+          alert-threshold: '100%'
+          comment-on-alert: false
+          fail-on-alert: false

.zizmor.yml

@@ -12,3 +12,11 @@ rules:
     config:
       policies:
         "*": hash-pin
+  dangerous-triggers:
+    # pr.yaml deliberately uses `pull_request_target` as a *gated* workflow: it
+    # runs from the trusted main branch, checks out PR code via refs/pull/*/head
+    # (never executing untrusted workflow YAML), and re-fetches analyzers/config
+    # from main so a malicious PR cannot disable checks. This is the canonical
+    # secure pattern, not the vulnerability the rule warns about.
+    ignore:
+      - pr.yaml

scripts/stryker-score.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""Compute the Stryker mutation score from mutation-report.json files and emit a
+github-action-benchmark `customBiggerIsBetter` payload for the gh-pages trend (#124).
+
+Aggregates every mutation-report.json under the given root (a repo may have more
+than one test project's report), applying Stryker's own score formula:
+
+    score = (Killed + Timeout) / (Killed + Timeout + Survived + NoCoverage) * 100
+
+Ignored / CompileError mutants are excluded from the denominator, exactly as
+Stryker does when it prints "The final mutation score".
+
+Usage: python scripts/stryker-score.py <search-root> <output-json>
+"""
+import glob
+import json
+import os
+import sys
+from collections import Counter
+
+DETECTED = ("Killed", "Timeout")
+UNDETECTED = ("Survived", "NoCoverage")
+
+
+def compute(root: str) -> float:
+    counts: Counter = Counter()
+    reports = glob.glob(os.path.join(root, "**", "mutation-report.json"), recursive=True)
+    if not reports:
+        raise SystemExit(f"No mutation-report.json found under {root!r}")
+    for path in reports:
+        with open(path, encoding="utf-8") as handle:
+            data = json.load(handle)
+        for file_data in data.get("files", {}).values():
+            for mutant in file_data.get("mutants", []):
+                counts[mutant["status"]] += 1
+
+    detected = sum(counts[s] for s in DETECTED)
+    denom = detected + sum(counts[s] for s in UNDETECTED)
+    score = round(100 * detected / denom, 2) if denom else 0.0
+    print(f"Mutation status counts: {dict(counts)}")
+    print(f"Mutation score: {score}% (from {len(reports)} report(s))")
+    return score
+
+
+def main(root: str, out_file: str) -> None:
+    score = compute(root)
+    payload = [{"name": "Mutation score", "unit": "%", "value": score}]
+    with open(out_file, "w", encoding="utf-8", newline="\n") as handle:
+        json.dump(payload, handle, indent=2)
+        handle.write("\n")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("usage: stryker-score.py <search-root> <output-json>", file=sys.stderr)
+        sys.exit(2)
+    main(sys.argv[1], sys.argv[2])

tests/Wolfgang.Etl.TestKit.Tests.Unit/stryker-config.json

@@ -0,0 +1,13 @@
+{
+  "stryker-config": {
+    "project": "Wolfgang.Etl.TestKit.csproj",
+    "target-framework": "net10.0",
+    "reporters": ["progress", "json", "html"],
+    "thresholds": {
+      "high": 90,
+      "low": 70,
+      "break": 70
+    },
+    "mutation-level": "Standard"
+  }
+}