feat(evaluation): add fuzz testing infrastructure and property-based tests

.cspell/general-technical.txt

@@ -62,6 +62,7 @@ asgi
 asgs
 assetguidance
 asyncio
+atheris
 attestations
 attunity
 audiologs
@@ -544,6 +545,7 @@ Hypervelocity
 hypervisor
 iaas
 ibm
+hypothesis
 ibpms
 icmp
 iconmonstr

.github/workflows/dataviewer-backend-pytests.yml

@@ -43,21 +43,21 @@ jobs:
         run: uv sync --extra dev --extra analysis --extra hdf5 --extra export --extra auth
 
       - name: Run pytest with coverage
-        run: uv run pytest -v --cov=src --cov-report=xml --cov-report=term-missing
+        run: uv run pytest -v --cov=src --cov-report=xml:../../../logs/coverage-dataviewer.xml --cov-report=term-missing
 
       - name: Upload coverage.xml artifact
         if: ${{ inputs.code-coverage && always() }}
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: pytest-dataviewer-coverage-xml
-          path: data-management/viewer/backend/coverage.xml
+          path: logs/coverage-dataviewer.xml
           retention-days: 30
 
       - name: Upload coverage to Codecov
         if: ${{ inputs.code-coverage && always() }}
         uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
         with:
-          files: coverage.xml
+          files: logs/coverage-dataviewer.xml
           use_oidc: true
           fail_ci_if_error: false
           verbose: true

.github/workflows/fuzz-regression-tests.yml

@@ -0,0 +1,59 @@
+name: Fuzz Regression Tests
+
+on:
+  workflow_call:
+    inputs:
+      code-coverage:
+        description: 'Enable Codecov coverage upload'
+        required: false
+        default: false
+        type: boolean
+
+permissions:
+  contents: read
+
+jobs:
+  fuzz-regression:
+    name: Fuzz Regression
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Setup Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57  # v8.0.0
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Run fuzz regression tests
+        run: uv run pytest tests/fuzz_harness.py -v --cov=tests --cov=data-management/viewer/backend/src --cov=training --cov-report=xml:logs/coverage-fuzz.xml --cov-report=term-missing
+
+      - name: Upload coverage.xml artifact
+        if: ${{ inputs.code-coverage && always() }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: pytest-fuzz-coverage-xml
+          path: logs/coverage-fuzz.xml
+          retention-days: 30
+
+      - name: Upload coverage to Codecov
+        if: ${{ inputs.code-coverage && always() }}
+        uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
+        with:
+          files: logs/coverage-fuzz.xml
+          use_oidc: true
+          fail_ci_if_error: false
+          verbose: true
+          flags: pytest-fuzz
+          name: pytest-fuzz-coverage

.github/workflows/main.yml

@@ -127,6 +127,16 @@ jobs:
       contents: read
       id-token: write
 
+  # Fuzz regression via deterministic corpus-based tests
+  fuzz-regression-tests:
+    name: Fuzz Regression Tests
+    uses: ./.github/workflows/fuzz-regression-tests.yml
+    with:
+      code-coverage: true
+    permissions:
+      contents: read
+      id-token: write
+
   # Python linting using ruff
   python-lint:
     name: Python Lint

.github/workflows/pr-validation.yml

@@ -147,6 +147,16 @@ jobs:
       contents: read
       id-token: write
 
+  # Fuzz regression via deterministic corpus-based tests
+  fuzz-regression-tests:
+    name: Fuzz Regression Tests
+    uses: ./.github/workflows/fuzz-regression-tests.yml
+    with:
+      code-coverage: true
+    permissions:
+      contents: read
+      id-token: write
+
   # Python linting using ruff
   python-lint:
     name: Python Lint

.github/workflows/pytest-tests.yml

@@ -44,14 +44,14 @@ jobs:
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: pytest-coverage-xml
-          path: coverage.xml
+          path: logs/coverage.xml
           retention-days: 30
 
       - name: Upload coverage to Codecov
         if: ${{ inputs.code-coverage && always() }}
         uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
         with:
-          files: coverage.xml
+          files: logs/coverage.xml
           use_oidc: true
           fail_ci_if_error: false
           verbose: true

codecov.yml

@@ -45,10 +45,35 @@ coverage:
           - vitest
         target: auto
         threshold: 1%
+      pytest-fuzz:
+        flags:
+          - pytest-fuzz
+        target: auto
+        threshold: 1%
     patch:
       default:
         target: auto
         threshold: 5%
+      go:
+        flags:
+          - go
+        informational: true
+      pester:
+        flags:
+          - pester
+        informational: true
+      pytest:
+        flags:
+          - pytest
+        informational: true
+      pytest-dataviewer:
+        flags:
+          - pytest-dataviewer
+        informational: true
+      vitest:
+        flags:
+          - vitest
+        informational: true
 
 flags:
   go:
@@ -61,7 +86,7 @@ flags:
     carryforward: true
   pytest:
     paths:
-      - src/
+      - training/
     carryforward: true
   pytest-dataviewer:
     paths:
@@ -75,6 +100,12 @@ flags:
     paths:
       - data-management/viewer/frontend/src/
     carryforward: true
+  pytest-fuzz:
+    paths:
+      - tests/
+      - data-management/viewer/backend/src/
+      - training/
+    carryforward: true
 
 parsers:
   jacoco:

data-management/viewer/backend/pyproject.toml

@@ -22,6 +22,7 @@
     "pytest-asyncio==1.3.0",
     "pytest-cov==7.1.0",
     "httpx==0.28.1",
+    "hypothesis>=6.100.0",
     "schemathesis==4.14.3",
 ]
 azure = [