test(fuzz): add coverage-guided + property-based fuzzing

.github/workflows/fuzz.yml

@@ -0,0 +1,92 @@
+name: Fuzz
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    # Weekly deeper run (longer per-target budget via FUZZ_SECONDS).
+    - cron: "41 4 * * 2"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: fuzz-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Always-on, cross-platform property tests. Fast, deterministic, no native deps.
+  property_tests:
+    name: Hypothesis property tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install fuzz dependencies
+        # Hash-pinned per OpenSSF Scorecard Pinned-Dependencies. Sources:
+        # fuzz/requirements-property.in (recompile with uv pip compile --generate-hashes).
+        run: python -m pip install --require-hashes -r fuzz/requirements-property.txt
+
+      - name: Run property-based fuzz tests
+        run: python -m pytest tests/fuzz -q
+
+  # Coverage-guided fuzzing with Atheris (Apache-2.0). Bounded per-target budget
+  # so CI stays cheap; schedule/dispatch runs use a longer budget.
+  coverage_guided:
+    name: Atheris coverage-guided
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        # Atheris ships wheels/builds cleanly for 3.11 (Clang + libFuzzer).
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install Atheris
+        # Hash-pinned per OpenSSF Scorecard Pinned-Dependencies. Sources:
+        # fuzz/requirements-atheris.in (recompile with uv pip compile --generate-hashes).
+        run: python -m pip install --require-hashes -r fuzz/requirements-atheris.txt
+
+      - name: Set per-target time budget
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "push" ]; then
+            echo "FUZZ_SECONDS=60" >> "$GITHUB_ENV"
+          else
+            echo "FUZZ_SECONDS=300" >> "$GITHUB_ENV"
+          fi
+
+      - name: Fuzz request-body parser
+        run: python fuzz/fuzz_request_body.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/request_body
+
+      - name: Fuzz agent-config parser
+        run: python fuzz/fuzz_agent_config.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/agent_config
+
+      - name: Fuzz secret redaction
+        run: python fuzz/fuzz_redaction.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/redaction
+
+      - name: Fuzz orchestration engine
+        run: python fuzz/fuzz_orchestration.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/orchestration
+
+      - name: Upload crash artifacts
+        if: failure()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4
+        with:
+          name: fuzz-crashes
+          path: crash-*
+          if-no-files-found: ignore

.gitignore

@@ -9,3 +9,6 @@ tempcred.txt
 
 # local codegraph index
 .codegraph/
+
+# hypothesis fuzzing DB
+.hypothesis/

conftest.py

@@ -0,0 +1,9 @@
+"""Pytest configuration for the repo root.
+
+The Atheris coverage-guided harnesses under ``fuzz/`` import ``atheris`` at
+module load time, which is only installed in the dedicated CI job. Ignore that
+directory during normal collection so the suite runs without the native
+toolchain. The Hypothesis property tests under ``tests/fuzz/`` are unaffected.
+"""
+
+collect_ignore = ["fuzz"]

docs/fuzzing.md

@@ -0,0 +1,67 @@
+# Fuzzing
+
+The orchestrator consumes untrusted input at a handful of well-defined seams:
+HTTP request bodies, agent-pool configuration, arbitrary prompt text, and trace
+payloads that pass through secret/PII redaction. Those seams are fuzzed with two
+complementary, permissively licensed tools.
+
+| Tool | License | Role |
+| --- | --- | --- |
+| [Hypothesis](https://hypothesis.readthedocs.io/) | MPL-2.0 | Always-on property tests in the normal `pytest` suite (`tests/fuzz/`). Deterministic, cross-platform, shrinks any counterexample to a minimal repro. |
+| [Atheris](https://github.com/google/atheris) | Apache-2.0 | Coverage-guided (libFuzzer) harnesses in `fuzz/`, run in a bounded CI job on Python 3.11. |
+
+Both drivers call the same invariant checks in [`fuzz/targets.py`](../fuzz/targets.py),
+so a bug found by either tool reproduces under the other.
+
+## Targets
+
+The surfaces were located with CodeGraph (`codegraph explore "parse decode
+deserialize request config validate untrusted input"`):
+
+1. **HTTP request body** — `server._coerce_json` / `_reject_unknown_keys` /
+   `_validate_mode` / `_validate_messages`. Arbitrary bytes must normalise to a
+   validated structure or raise `RequestError` / a JSON decode error — never an
+   unhandled crash.
+2. **Agent config** — `orchestrator.ModelAgent.from_dict`. Arbitrary decoded
+   JSON must yield a well-typed `ModelAgent` or raise `KeyError`/`TypeError`/
+   `ValueError`.
+3. **Secret/PII redaction** — `orchestrator.redact_text` / `redact_value`.
+   Never crashes, always returns `str`, is **idempotent** (re-redacting redacted
+   text is a no-op), and preserves container shape.
+4. **End-to-end orchestration** — `orchestrator.TaskOrchestrator.run` against
+   `mock://` providers (fully offline). Arbitrary prompt text and mode must
+   produce a JSON-serialisable record whose SSE framing round-trips.
+
+## Running locally
+
+Property tests (no native toolchain needed):
+
+```bash
+pip install -e '.[fuzz]'      # hypothesis
+pytest tests/fuzz -q
+```
+
+Coverage-guided harnesses (needs Clang/libFuzzer; use Python < 3.13):
+
+```bash
+pip install atheris
+python fuzz/fuzz_request_body.py  -max_total_time=60 fuzz/corpus/request_body
+python fuzz/fuzz_agent_config.py  -max_total_time=60 fuzz/corpus/agent_config
+python fuzz/fuzz_redaction.py     -max_total_time=60 fuzz/corpus/redaction
+python fuzz/fuzz_orchestration.py -max_total_time=60 fuzz/corpus/orchestration
+```
+
+Seed corpora live in `fuzz/corpus/<target>/`.
+
+## CI
+
+`.github/workflows/fuzz.yml` runs the property tests on every push/PR and the
+Atheris harnesses with a 60s-per-target budget on PRs (300s on the weekly
+schedule / manual dispatch) to keep CI cost bounded. Crash inputs are uploaded
+as artifacts.
+
+## Background
+
+For the theory behind coverage-guided greybox fuzzing, see
+[`papers/fuzzing-art-science-engineering-manes-2019.pdf`](papers/fuzzing-art-science-engineering-manes-2019.pdf)
+(Manès et al., *The Art, Science, and Engineering of Fuzzing: A Survey*).

fuzz/__init__.py

@@ -0,0 +1,7 @@
+"""Fuzzing harnesses and shared fuzz-target logic for contextual-orchestrator.
+
+The heavy lifting lives in :mod:`fuzz.targets`, which exposes one ``exercise_*``
+function per untrusted-input surface. Both the Atheris coverage-guided harnesses
+(``fuzz/fuzz_*.py``) and the Hypothesis property tests (``tests/fuzz/``) call the
+same functions, so a bug found by either tool reproduces under the other.
+"""

fuzz/corpus/agent_config/bad_priority.json

@@ -0,0 +1 @@
+{"id":"good_agent","model":"m","priority":"not-an-int"}

fuzz/corpus/agent_config/bad_single_word_id.json

@@ -0,0 +1 @@
+{"id":"oneword","model":"m"}

fuzz/corpus/agent_config/legacy_exclusion.json

@@ -0,0 +1 @@
+{"id":"builder_agent","model":"m","provider_exclusion":["openai"],"disabled":true}

fuzz/corpus/agent_config/missing_id.json

@@ -0,0 +1 @@
+{"model":"m"}

fuzz/corpus/agent_config/valid_agent.json

@@ -0,0 +1 @@
+{"id":"general_agent","model":"mock-generalist","base_url":"mock://x","tags":["reasoning"],"priority":1}

fuzz/corpus/orchestration/coding.txt

@@ -0,0 +1 @@
+debug this code and implement a fix with tests

fuzz/corpus/orchestration/complex_workflow.txt

@@ -0,0 +1 @@
+please analyze the paper and verify the security architecture

fuzz/corpus/orchestration/korean.txt

@@ -0,0 +1 @@
+안녕하세요 구현 분석

fuzz/corpus/orchestration/short_route.txt

@@ -0,0 +1 @@
+hi

fuzz/corpus/redaction/apikey_and_email.txt

@@ -0,0 +1 @@
+my api_key = ABCDEF1234567890 and email bob@example.com

fuzz/corpus/redaction/bearer.txt

@@ -0,0 +1 @@
+authorization: Bearer sk-abcdef1234567890TOKEN

fuzz/corpus/redaction/multi_secret.txt

@@ -0,0 +1 @@
+token="EXAMPLE-fuzz-seed-000000000000" password: hunter2hunter2

fuzz/corpus/redaction/plain.txt

@@ -0,0 +1 @@
+nothing secret here, just plain text

fuzz/corpus/request_body/bad_message.json

@@ -0,0 +1 @@
+{"messages":[{"role":"root","content":123}]}

fuzz/corpus/request_body/bad_mode_unknown_field.json

@@ -0,0 +1 @@
+{"run_mode":"bogus","extra":1}

fuzz/corpus/request_body/json_array.json

@@ -0,0 +1 @@
+[1,2,3]

fuzz/corpus/request_body/nested.json

@@ -0,0 +1 @@
+{"nested":{"a":{"b":{"c":[1,2,3]}}}}

fuzz/corpus/request_body/not_json.txt

@@ -0,0 +1 @@
+not json at all

fuzz/corpus/request_body/valid_conduct.json

@@ -0,0 +1 @@
+{"messages":[{"role":"user","content":"analyze the security architecture"}],"run_mode":"conduct"}

fuzz/corpus/request_body/valid_route.json

@@ -0,0 +1 @@
+{"prompt_text":"summarize this text","run_mode":"route"}

fuzz/fuzz_agent_config.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""Atheris coverage-guided harness: agent-pool config parser.
+
+Surface: ``orchestrator.ModelAgent.from_dict`` -- parses each entry of an
+untrusted ``agents.json`` config file.
+
+Run locally::
+
+    python fuzz/fuzz_agent_config.py -max_total_time=60 fuzz/corpus/agent_config
+"""
+
+import json
+import sys
+
+import atheris
+
+with atheris.instrument_imports():
+    from fuzz.targets import exercise_agent_config
+
+
+def one_input(data: bytes) -> None:
+    fdp = atheris.FuzzedDataProvider(data)
+    text = fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes())
+    try:
+        value = json.loads(text)
+    except (ValueError, RecursionError):
+        return
+    exercise_agent_config(value)
+
+
+def main() -> None:
+    atheris.Setup(sys.argv, one_input)
+    atheris.Fuzz()
+
+
+if __name__ == "__main__":
+    main()

fuzz/fuzz_orchestration.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+"""Atheris coverage-guided harness: end-to-end orchestration on arbitrary prompt.
+
+Surface: ``orchestrator.TaskOrchestrator.run`` against ``mock://`` providers --
+drives prompt classification, agent scoring, route/conduct, trace assembly, and
+SSE framing entirely offline.
+
+Run locally::
+
+    python fuzz/fuzz_orchestration.py -max_total_time=60 fuzz/corpus/orchestration
+"""
+
+import sys
+
+import atheris
+
+with atheris.instrument_imports():
+    from fuzz.targets import exercise_orchestration
+
+_MODES = ("auto", "route", "conduct")
+
+
+def one_input(data: bytes) -> None:
+    fdp = atheris.FuzzedDataProvider(data)
+    mode = _MODES[fdp.ConsumeIntInRange(0, len(_MODES) - 1)]
+    prompt = fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes())
+    exercise_orchestration(prompt, mode)
+
+
+def main() -> None:
+    atheris.Setup(sys.argv, one_input)
+    atheris.Fuzz()
+
+
+if __name__ == "__main__":
+    main()

fuzz/fuzz_redaction.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+"""Atheris coverage-guided harness: secret/PII redaction.
+
+Surface: ``orchestrator.redact_text`` / ``redact_value`` -- regex + recursive
+masking applied to arbitrary trace payloads before they leave the process.
+Idempotence and no-crash are the load-bearing invariants (see ``fuzz.targets``).
+
+Run locally::
+
+    python fuzz/fuzz_redaction.py -max_total_time=60 fuzz/corpus/redaction
+"""
+
+import sys
+
+import atheris
+
+with atheris.instrument_imports():
+    from fuzz.targets import exercise_redaction
+
+
+def one_input(data: bytes) -> None:
+    fdp = atheris.FuzzedDataProvider(data)
+    text = fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes())
+    exercise_redaction(text)
+
+
+def main() -> None:
+    atheris.Setup(sys.argv, one_input)
+    atheris.Fuzz()
+
+
+if __name__ == "__main__":
+    main()

fuzz/fuzz_request_body.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+"""Atheris coverage-guided harness: HTTP request-body parser + validators.
+
+Surface: ``server._coerce_json`` / ``_reject_unknown_keys`` / ``_validate_mode``
+/ ``_validate_messages`` -- everything that touches an untrusted HTTP body.
+
+Run locally (needs a permissive-licensed build of Atheris, Apache-2.0)::
+
+    python fuzz/fuzz_request_body.py -atomic_step -max_total_time=60 fuzz/corpus/request_body
+"""
+
+import sys
+
+import atheris
+
+with atheris.instrument_imports():
+    from fuzz.targets import exercise_request_body
+
+
+def one_input(data: bytes) -> None:
+    exercise_request_body(data)
+
+
+def main() -> None:
+    atheris.Setup(sys.argv, one_input)
+    atheris.Fuzz()
+
+
+if __name__ == "__main__":
+    main()

fuzz/requirements-atheris.in

@@ -0,0 +1,3 @@
+# Atheris coverage-guided job deps (Python 3.11). Compile: uv pip compile fuzz/requirements-atheris.in --generate-hashes --python-version 3.11 --universal -o fuzz/requirements-atheris.txt
+pip
+atheris>=2.3