feat(nim): add evidence-grade all-model benchmark (#86)

.github/workflows/ci.yml

@@ -36,3 +36,51 @@ jobs:
 
       - name: Run full test suite
         run: uv run --locked --extra api --extra db --extra queue --group dev python -m pytest -q
+
+  nim_benchmark_quality:
+    name: NIM benchmark coverage, docstrings, and package smoke
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install hash-locked quality tools
+        run: |
+          python -m pip install --require-hashes -r requirements.lock
+          python -m pip install --require-hashes -r requirements-opencode-review-ci.txt
+
+      - name: Prove complete benchmark coverage and public docstrings
+        run: |
+          set -euo pipefail
+          python -m coverage erase
+          python -m coverage run --branch \
+            --source=contextual_orchestrator.nim_benchmark \
+            -m pytest \
+            tests/test_nim_benchmark.py \
+            tests/test_nim_benchmark_release_acceptance.py \
+            tests/test_nim_benchmark_workflow_contract.py \
+            -q
+          python -m coverage report \
+            --include=contextual_orchestrator/nim_benchmark.py \
+            --show-missing \
+            --fail-under=100
+          python -m interrogate -f 100 contextual_orchestrator/nim_benchmark.py
+
+      - name: Build, install, and import the wheel
+        run: |
+          set -euo pipefail
+          rm -rf dist "$RUNNER_TEMP/nim-wheel-site"
+          python -m pip wheel --no-deps --no-build-isolation . --wheel-dir dist
+          python -m pip install --no-deps \
+            --target "$RUNNER_TEMP/nim-wheel-site" \
+            dist/contextual_orchestrator-*.whl
+          cd "$RUNNER_TEMP"
+          PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \
+            python -c "import contextual_orchestrator; import contextual_orchestrator.nim_benchmark"

.github/workflows/fuzz.yml

@@ -93,6 +93,9 @@ jobs:
       - name: Fuzz reasoning-effort profile parser
         run: python fuzz/fuzz_reasoning_effort_profile.py -max_total_time="${FUZZ_SECONDS}" -artifact_prefix=crash- fuzz/corpus/reasoning_effort_profile
 
+      - name: Fuzz NIM model-catalog parser
+        run: python fuzz/fuzz_nim_catalog.py -max_total_time="${FUZZ_SECONDS}" -artifact_prefix=crash- fuzz/corpus/nim_catalog
+
       - name: Upload crash artifacts
         if: failure()
         uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5

.github/workflows/nim-benchmark.yml

@@ -0,0 +1,151 @@
+name: NIM benchmark
+
+# Evidence-grade NVIDIA NIM model discovery and cost-quality benchmark.
+# Manual dispatch defaults to a deterministic dry run. Monthly scheduled runs
+# are live but use a conservative hard request cap. The first-of-month schedule
+# keeps the next run inside the current reviewed evidence window; stale evidence
+# still fails closed. Dry execution receives no provider credential; only the
+# live step can read NVIDIA_NIM_API_KEY.
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Dry run without contacting NVIDIA"
+        type: boolean
+        default: true
+      max_total_requests:
+        description: "Hard cap on provider requests for this run"
+        type: number
+        default: 2000
+      pricing_scenario:
+        description: "Optional reviewed pricing-scenario JSON path"
+        type: string
+        default: ""
+  schedule:
+    - cron: "23 3 1 * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: nim-benchmark
+  cancel-in-progress: false
+
+jobs:
+  dry_run_benchmark:
+    name: Deterministic NIM benchmark dry run
+    if: github.event_name == 'workflow_dispatch' && inputs.dry_run == true
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install pinned runtime
+        run: |
+          python -m pip install --require-hashes -r requirements.lock
+          python -m pip install --no-deps -e .
+
+      - name: Run dry benchmark
+        env:
+          MAX_REQUESTS: ${{ inputs.max_total_requests }}
+          PRICING_SCENARIO: ${{ inputs.pricing_scenario }}
+          PROVENANCE_GIT_SHA: ${{ github.sha }}
+          PROVENANCE_RUN_ID: ${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          extra_args=()
+          if [ -n "$PRICING_SCENARIO" ]; then
+            extra_args+=(--pricing-scenario "$PRICING_SCENARIO")
+          fi
+          python -m contextual_orchestrator nim-benchmark \
+            --dry-run \
+            "${extra_args[@]}" \
+            --task-manifest examples/nim_task_manifest.json \
+            --output-dir benchmark_artifacts \
+            --max-total-requests "$MAX_REQUESTS" \
+            --git-sha "$PROVENANCE_GIT_SHA" \
+            --workflow-run-id "$PROVENANCE_RUN_ID"
+
+      - name: Upload dry-run artifacts
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5
+        with:
+          name: nim-benchmark-dry-${{ github.run_id }}
+          path: benchmark_artifacts/
+          retention-days: 30
+          if-no-files-found: error
+
+  live_benchmark:
+    name: Live NIM catalog benchmark
+    if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.dry_run != true)
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install pinned runtime
+        run: |
+          python -m pip install --require-hashes -r requirements.lock
+          python -m pip install --no-deps -e .
+
+      - name: Resolve live parameters
+        id: live_params
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_MAX_REQUESTS: ${{ inputs.max_total_requests }}
+          INPUT_PRICING: ${{ inputs.pricing_scenario }}
+        run: |
+          set -euo pipefail
+          if [ "$EVENT_NAME" = "schedule" ]; then
+            echo "max_requests=2000" >> "$GITHUB_OUTPUT"
+            echo "pricing_scenario=" >> "$GITHUB_OUTPUT"
+          else
+            echo "max_requests=${INPUT_MAX_REQUESTS}" >> "$GITHUB_OUTPUT"
+            echo "pricing_scenario=${INPUT_PRICING}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run live benchmark
+        env:
+          NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}
+          MAX_REQUESTS: ${{ steps.live_params.outputs.max_requests }}
+          PRICING_SCENARIO: ${{ steps.live_params.outputs.pricing_scenario }}
+          PROVENANCE_GIT_SHA: ${{ github.sha }}
+          PROVENANCE_RUN_ID: ${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          extra_args=()
+          if [ -n "$PRICING_SCENARIO" ]; then
+            extra_args+=(--pricing-scenario "$PRICING_SCENARIO")
+          fi
+          python -m contextual_orchestrator nim-benchmark \
+            "${extra_args[@]}" \
+            --task-manifest examples/nim_task_manifest.json \
+            --output-dir benchmark_artifacts \
+            --max-total-requests "$MAX_REQUESTS" \
+            --git-sha "$PROVENANCE_GIT_SHA" \
+            --workflow-run-id "$PROVENANCE_RUN_ID"
+
+      - name: Upload live benchmark artifacts
+        if: always()
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5
+        with:
+          name: nim-benchmark-live-${{ github.run_id }}
+          path: benchmark_artifacts/
+          retention-days: 90
+          if-no-files-found: warn

.gitignore

@@ -14,3 +14,6 @@ tempcred.txt
 .hypothesis/
 .secrets/
 .coverage
+
+# local benchmark artifacts (uploaded by CI, not committed)
+benchmark_artifacts/

CHANGELOG.md

@@ -12,6 +12,8 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
 
 ### Fixed
 
+- NVIDIA NIM benchmark cells now reconcile reported prompt and completion
+  usage independently, preventing negative completion counts after failures.
 - OpenRouter discovery no longer marks the entire credential account
   evidence-only. Authenticated catalog rows may serve ordinary requests, while
   ZDR-only requests still require explicit route-level ZDR evidence.
@@ -114,12 +116,19 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
   `role_effort_catalog=default_role_effort_catalog()` to attach the same
   `reasoning_effort_snapshot` on `complete`, `run`, `stream_route`, and
   `batch_route`; omit it to keep today's payload.
+- Add an optional provider-neutral NVIDIA NIM benchmark harness that dynamically discovers the live `/v1/models` catalog, probes every discovered model under bounded concurrency and a hard request cap, records machine-readable capability outcomes, and compares direct, route-once, bounded-conduct, and explicit pricing-scenario policies over a locked task manifest.
+- Add deterministic no-egress benchmark dry runs, secret-redacted JSON/CSV/Markdown evidence artifacts, paired bootstrap uncertainty, quality-latency and quality-hypothetical-cost Pareto frontiers, all-modality catalog fuzzing, and a manually gated benchmark workflow.
+- Add a validated deterministic one-frame H.264 MP4 probe fixture, complete preflight reservation for every discovered model-capability cell plus the full evaluation envelope, and explicit evidence-sufficiency fields that keep the bundled smoke manifest from authorizing production routing.
+- Add direct benchmark quality gates for 100% production statement/branch coverage, 100% public docstrings, wheel build/install/import smoke testing, and optional-import isolation.
 - Streamed `/v1/responses` workflow runs now request provider usage only from
   agents explicitly marked `stream_usage_supported`, preserve provider-declared
   SSE usage, record per-step `stream` cost-ledger rows, and expose cost status
   plus usage-record identities. Missing provider usage is explicitly
   unavailable; the gateway does not estimate billing tokens from the final
   answer, and nested gateway upstreams remain compatible (ADR 0040).
+- Experimental CEFR criterion-observation gateway with exact contract checks,
+  independent rater blindness, bounded structured-output parsing, replay
+  provenance, and human-review routing; it emits no final CEFR level or score.
 
 ### Fixed
 
@@ -338,6 +347,31 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
 - Strix B105 false positives eliminated at the source: KV credential-name
   constants renamed `*_CREDENTIAL_NAME`; readiness label keys renamed
   `readiness_ok/warning/failure`. (#833)
+- Expose a stable complete-run request planning view and align API, CLI, manual-workflow, and deterministic test caps with the locked thirty-task evidence floor, preserving fail-before-probe behavior.
+- Fail closed after catalog discovery but before capability egress when the complete all-model probe and equal-budget evaluation plan cannot fit the configured hard request cap; the monthly 2,000-request ceiling covers the representative 127-model, thirty-task, seven-worker plan requiring 1,924 requests, including route-once's full equal-call envelope and direct-cell judge calls, and still rejects larger plans before partial probing.
+- Align the monthly NIM live schedule with the reviewed access-cost evidence window while preserving fail-closed behavior after its validity horizon.
+- Scale the equal NIM policy-cell token budget with the five-call envelope so the conduct arm can carry its prompts while every policy retains the same total allowance.
+- Treat provider HTTP 401 and 403 responses during capability probes as authentication rejection, and keep live evaluation on the same DNS-pinned benchmark transport used by discovery and probes.
+- Pin Atheris by Python interpreter so the Python 3.11 fuzz job and the newer central coverage-evidence image both install a published, hash-locked wheel.
+- Record the reviewed current NVIDIA NIM General FAQ as expiring evidence for free Developer Program hosted-endpoint prototyping access, while keeping NVIDIA AI Enterprise production licensing and every hypothetical model rate explicitly separate.
+- Require live hypothetical pricing scenarios to carry reviewed source, reviewer, review date, validity horizon, rate basis, uncertainty, and explicit rates; reject unreviewed, incomplete, future-dated, or expired price evidence before provider egress.
+- Give direct, route-once, conduct, and reviewed cheapest-worker cells one equal total prompt-plus-completion token budget and one common five-call envelope, with configured-versus-observed evidence in every cell.
+- Keep the optional NIM adapter lazy: importing the runtime package no longer imports the benchmark or mutates benchmark globals.
+- Record immutable source-artifact digests and exact Git tree identity in the integration evidence so buyers and reviewers can reproduce the accepted benchmark source independently of transient workflow state.
+
+- NIM benchmark provider responses are bounded to 8 MiB, and live HTTPS
+  requests use validation-time public-address pinning with original-host TLS,
+  no proxy lookup, and no redirect following.
+- Live pricing evidence is rejected unless its source, reviewer, dates, rate
+  basis, uncertainty, and explicit rates are complete and current.
+- Direct, route-once, conduct, and reviewed cheapest-worker cells share one
+  total token budget and five-call envelope, with configured and observed
+  values recorded separately.
+- Complete catalog probing and the full evaluation reserve are planned before
+  capability egress; the benchmark fails closed when the configured cap is too
+  small, and the scheduled workflow uses a reviewed 2,000-request ceiling.
+- The NIM access-cost evidence, hypothetical pricing provenance, and source
+  artifact digests remain explicit and independently reproducible.
 
 ### Security
 
@@ -349,6 +383,9 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
 - Worker-agent pool boundaries are enforced beside object lookup so a
   different-pool id can no longer read or mutate another pool's agent.
 
+- Provider hosts resolving to any non-globally-routable address are rejected,
+  including RFC 6598 shared space; benchmark artifacts refuse secret leakage.
+
 ### References
 
 - Sakana AI. (2026). *Sakana Fugu Technical Report*.

README.md

@@ -296,9 +296,35 @@ is read from a **KV config store**, never `os.getenv`.
   `pg_tiktoken` counting, and the production batch backend without adding a
   repository split here.
 
-Grounding papers (LLM cost, routing, load balancing) live in
+Grounding papers (LLM cost, routing, load balancing, evaluation) live in
 [docs/papers](docs/papers/README.md) with citations.
 
+### NIM cost-quality benchmark (optional harness)
+
+Evidence-grade benchmark of the routing policies against a **dynamically
+discovered** NVIDIA NIM catalog. It probes chat, completions, Responses,
+embeddings, image/video/audio understanding, transcription, and speech; compares
+direct, route-once, and bounded-conduct cells under one equal total-token and
+call budget; records paired uncertainty and Pareto frontiers; and keeps reviewed
+actual endpoint-access evidence separate from optional hypothetical paid rates.
+The bundled manifest is smoke-sized and reports `evidence_review_required` only
+after its paired cells complete; `routing_recommendation` remains null and no
+benchmark artifact automatically changes production routing.
+
+The adapter is lazy and optional: ordinary `import contextual_orchestrator` does
+not import or mutate it. Deterministic `--dry-run` receives no network access or
+NVIDIA secret. Live execution resolves `NVIDIA_NIM_API_KEY` from the credential
+registry, pins HTTPS connections to validation-time public addresses, rejects
+redirects and proxy routing, and fails closed on missing/expired evidence. See
+[docs/nim_benchmark.md](docs/nim_benchmark.md) and the
+[engineering decision record](docs/doctoring/nim-benchmark-evidence-grade.md).
+
+```bash
+python -m contextual_orchestrator nim-benchmark --dry-run \
+  --pricing-scenario examples/nim_pricing_scenario.json \
+  --output-dir benchmark_artifacts
+```
+
 ## Design Artifacts
 
 - [Library research](docs/library_research.md)
@@ -363,6 +389,7 @@ python -m pytest -q tests/test_reasoning_effort_profile.py
 python tests/test_admin_contract.py
 python tests/test_conventions.py
 python tests/test_api_contract.py
+python tests/test_nim_benchmark.py
 python tests/test_security_hardening.py
 python tests/test_chat_model_capability_isolation.py
 python tests/test_chat_transport_role_separation.py

conductor/tracks.md

@@ -5,3 +5,4 @@
 | 001-paper-grounded-orchestrator | active | Implement the source-backed orchestration contract with TDD, DDD, and CDD |
 | 002-enterprise-design-foundation | active | Add paper-grounded screen design, user stories, REST API, code/DB conventions, and i18n |
 | 003-reasoning-effort-profiles | active | Issue #568: versioned per-role `reasoning_effort_profile`, equal-budget θ̂ RMSE ablation, snapshot on run/stream/batch, production defaults locked |
+| 004-nim-cost-quality-benchmark | active | Evidence-grade NIM catalog discovery, all-modality capability probes, and the route/conduct/single-worker cost-quality benchmark (docs/nim_benchmark.md) |

contextual_orchestrator/__main__.py

@@ -494,6 +494,13 @@ def main(argv: list[str] | None = None) -> None:
         _check_fast_mlsirm_command()
         return
 
+    if arguments and arguments[0] == "nim-benchmark":
+        # Optional benchmark harness (issue #86): dynamic NIM catalog discovery,
+        # all-modality capability probes, and the cost-quality policy benchmark.
+        from .nim_benchmark import run_benchmark_cli
+
+        sys.exit(run_benchmark_cli(arguments[1:]))
+
     parser = argparse.ArgumentParser(description="Route or conduct chat requests across model agents.")
     parser.add_argument("prompt", nargs="?", help="User prompt for CLI mode.")
     parser.add_argument("--agents", default="examples/agents.mock.json", help="Agent config JSON.")