diff --git a/.github/workflows/actions-audit.yaml b/.github/workflows/actions-audit.yaml index e3efc27..123e78f 100644 --- a/.github/workflows/actions-audit.yaml +++ b/.github/workflows/actions-audit.yaml @@ -48,6 +48,11 @@ jobs: run: go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 - name: Run actionlint + env: + # Gate on shellcheck warning+ only. The info/style nits (SC2012 "use + # find not ls", SC2035 "use ./*glob*") in the canonical pr.yaml are + # not worth failing every PR over; warnings and errors still gate. + SHELLCHECK_OPTS: --severity=warning run: actionlint -color zizmor: @@ -76,9 +81,9 @@ jobs: GH_TOKEN: ${{ github.token }} # Report-only for now: don't fail the job on findings, upload them to # Code Scanning instead. Remove `|| true` to turn this into a gate. - run: zizmor --format sarif .github/workflows/ > zizmor.sarif || true + run: zizmor --config .zizmor.yml --format sarif .github/workflows/ > zizmor.sarif || true - name: Upload zizmor SARIF - uses: github/codeql-action/upload-sarif@e0647621c2984b5ed2f768cb892365bf2a616ad1 # v4 + uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4 with: sarif_file: zizmor.sarif diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 29a43fc..7309a7b 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -77,7 +77,7 @@ jobs: - name: Initialize CodeQL if: steps.check-csharp.outputs.has-csharp == 'true' - uses: github/codeql-action/init@e0647621c2984b5ed2f768cb892365bf2a616ad1 # v4 + uses: github/codeql-action/init@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4 with: languages: ${{ matrix.language }} # security-extended adds the broader security query pack on top of the @@ -159,7 +159,7 @@ jobs: - name: Perform CodeQL Analysis id: perform-codeql-analysis if: steps.check-csharp.outputs.has-csharp == 'true' - uses: github/codeql-action/analyze@e0647621c2984b5ed2f768cb892365bf2a616ad1 # v4 + uses: github/codeql-action/analyze@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 177debb..a9e6b3a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -275,7 +275,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: refs/pull/${{ github.event.pull_request.number }}/head persist-credentials: false @@ -337,7 +337,7 @@ jobs: echo "✅ Configuration files secured - using versions from main branch" - name: Setup .NET - uses: actions/setup-dotnet@v5 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6 with: # Match the test stages' SDK list so the solution's older-TFM projects # (test projects target netcoreapp3.1 / net5.0-net7.0) restore and build @@ -394,7 +394,7 @@ jobs: --no-build - name: Upload SARIF to Code Scanning - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4 with: sarif_file: inspect.sarif diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml index 38e7269..5246e26 100644 --- a/.github/workflows/scorecard.yaml +++ b/.github/workflows/scorecard.yaml @@ -45,6 +45,6 @@ jobs: publish_results: true - name: Upload SARIF to Code Scanning - uses: github/codeql-action/upload-sarif@e0647621c2984b5ed2f768cb892365bf2a616ad1 # v4 + uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4 with: sarif_file: results.sarif diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 54c85f3..3845a86 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -57,6 +57,6 @@ jobs: --error || true - name: Upload Semgrep SARIF - uses: github/codeql-action/upload-sarif@e0647621c2984b5ed2f768cb892365bf2a616ad1 # v4 + uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4 with: sarif_file: semgrep.sarif diff --git a/.github/workflows/stryker.yaml b/.github/workflows/stryker.yaml index 5eafc46..5f8d68f 100644 --- a/.github/workflows/stryker.yaml +++ b/.github/workflows/stryker.yaml @@ -9,6 +9,11 @@ # - workflow_dispatch: manual ad-hoc runs (e.g. while iterating on tests) # - schedule (weekly Sunday 06:00 UTC): catch quality regressions between # releases without burning CI time on every PR (mutation testing is slow) +# - pull_request (src/** or tests/** changes): the #124 enforcement gate. The +# per-project stryker-config.json pins target-framework:net10.0, so a PR runs +# a single-TFM mutation pass (~minutes) rather than the full matrix, and the +# config's `break` threshold fails the PR if the score regresses below floor. +# - push to main: re-measure and publish the score to the gh-pages trend. # # Stryker discovers each test project's mutation targets via the standard # project-reference graph. Two configuration modes are supported: @@ -23,6 +28,17 @@ on: workflow_dispatch: schedule: - cron: '0 6 * * 0' # weekly Sunday 06:00 UTC + pull_request: + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/stryker.yaml' + - 'scripts/stryker-score.py' + push: + branches: [main] + paths: + - 'src/**' + - 'tests/**' permissions: contents: read @@ -102,8 +118,20 @@ jobs: $tests | ForEach-Object { Write-Host " $($_.FullName)" } Write-Host "" + # Opt-in gating (#124): only run test projects that ship a + # stryker-config.json. A project without one has no documented + # score floor yet, so running it with Stryker defaults would be an + # ungated, all-TFM pass that is slow on every PR and can't fail the + # gate meaningfully. Add a stryker-config.json next to a test + # project to bring it under the gate. + $ran = 0 foreach ($proj in $tests) { $dir = $proj.DirectoryName + if (-not (Test-Path (Join-Path $dir 'stryker-config.json'))) { + Write-Host "::notice::No stryker-config.json in $dir - skipping (add one to enable the mutation gate for this project)." + continue + } + $ran++ Write-Host "::group::Stryker in $dir" Push-Location $dir try { @@ -114,6 +142,11 @@ jobs: } Write-Host "::endgroup::" } + + if ($ran -eq 0) { + Write-Host "::notice::No test project has a stryker-config.json - nothing to gate." + exit 0 + } } if ($failed -gt 0) { @@ -130,3 +163,51 @@ jobs: **/StrykerOutput/** if-no-files-found: ignore retention-days: 30 + + # Publish the mutation score to the gh-pages trend (#124), charted by the same + # benchmark-action pattern the BDN benchmark trend uses. Separate job so only + # it carries `contents: write`; the gate job above stays least-privilege. Runs + # only on trusted, non-PR events (a fork PR must never push to gh-pages) and + # only when the gate above passed (needs: stryker). + publish-trend: + name: Publish mutation-score trend + needs: stryker + if: github.event_name == 'push' || github.event_name == 'schedule' + runs-on: ubuntu-latest + permissions: + contents: write # push the score history to the gh-pages branch + steps: + - name: Check out repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Setup Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.x' + + - name: Download Stryker report + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: stryker-report-${{ github.run_id }} + path: stryker-report + + - name: Extract mutation score + run: python scripts/stryker-score.py stryker-report stryker-score.json + + - name: Publish score to gh-pages trend + uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1 + with: + name: Mutation score + tool: 'customBiggerIsBetter' + output-file-path: stryker-score.json + gh-pages-branch: gh-pages + benchmark-data-dir-path: dev/stryker + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Bigger-is-better: alert if the score drops. Report-only (the PR gate + # above is the hard enforcement); this line just annotates the trend. + alert-threshold: '100%' + comment-on-alert: false + fail-on-alert: false diff --git a/.zizmor.yml b/.zizmor.yml index f9d2374..efcc5db 100644 --- a/.zizmor.yml +++ b/.zizmor.yml @@ -12,3 +12,11 @@ rules: config: policies: "*": hash-pin + dangerous-triggers: + # pr.yaml deliberately uses `pull_request_target` as a *gated* workflow: it + # runs from the trusted main branch, checks out PR code via refs/pull/*/head + # (never executing untrusted workflow YAML), and re-fetches analyzers/config + # from main so a malicious PR cannot disable checks. This is the canonical + # secure pattern, not the vulnerability the rule warns about. + ignore: + - pr.yaml diff --git a/scripts/stryker-score.py b/scripts/stryker-score.py new file mode 100644 index 0000000..f242ccb --- /dev/null +++ b/scripts/stryker-score.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Compute the Stryker mutation score from mutation-report.json files and emit a +github-action-benchmark `customBiggerIsBetter` payload for the gh-pages trend (#124). + +Aggregates every mutation-report.json under the given root (a repo may have more +than one test project's report), applying Stryker's own score formula: + + score = (Killed + Timeout) / (Killed + Timeout + Survived + NoCoverage) * 100 + +Ignored / CompileError mutants are excluded from the denominator, exactly as +Stryker does when it prints "The final mutation score". + +Usage: python scripts/stryker-score.py +""" +import glob +import json +import os +import sys +from collections import Counter + +DETECTED = ("Killed", "Timeout") +UNDETECTED = ("Survived", "NoCoverage") + + +def compute(root: str) -> float: + counts: Counter = Counter() + reports = glob.glob(os.path.join(root, "**", "mutation-report.json"), recursive=True) + if not reports: + raise SystemExit(f"No mutation-report.json found under {root!r}") + for path in reports: + with open(path, encoding="utf-8") as handle: + data = json.load(handle) + for file_data in data.get("files", {}).values(): + for mutant in file_data.get("mutants", []): + counts[mutant["status"]] += 1 + + detected = sum(counts[s] for s in DETECTED) + denom = detected + sum(counts[s] for s in UNDETECTED) + score = round(100 * detected / denom, 2) if denom else 0.0 + print(f"Mutation status counts: {dict(counts)}") + print(f"Mutation score: {score}% (from {len(reports)} report(s))") + return score + + +def main(root: str, out_file: str) -> None: + score = compute(root) + payload = [{"name": "Mutation score", "unit": "%", "value": score}] + with open(out_file, "w", encoding="utf-8", newline="\n") as handle: + json.dump(payload, handle, indent=2) + handle.write("\n") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("usage: stryker-score.py ", file=sys.stderr) + sys.exit(2) + main(sys.argv[1], sys.argv[2]) diff --git a/tests/Wolfgang.Etl.TestKit.Tests.Unit/stryker-config.json b/tests/Wolfgang.Etl.TestKit.Tests.Unit/stryker-config.json new file mode 100644 index 0000000..0e5e611 --- /dev/null +++ b/tests/Wolfgang.Etl.TestKit.Tests.Unit/stryker-config.json @@ -0,0 +1,13 @@ +{ + "stryker-config": { + "project": "Wolfgang.Etl.TestKit.csproj", + "target-framework": "net10.0", + "reporters": ["progress", "json", "html"], + "thresholds": { + "high": 90, + "low": 70, + "break": 70 + }, + "mutation-level": "Standard" + } +}