diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac316b61d..bf593bb0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,3 +36,51 @@ jobs: - name: Run full test suite run: uv run --locked --extra api --extra db --extra queue --group dev python -m pytest -q + + nim_benchmark_quality: + name: NIM benchmark coverage, docstrings, and package smoke + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install hash-locked quality tools + run: | + python -m pip install --require-hashes -r requirements.lock + python -m pip install --require-hashes -r requirements-opencode-review-ci.txt + + - name: Prove complete benchmark coverage and public docstrings + run: | + set -euo pipefail + python -m coverage erase + python -m coverage run --branch \ + --source=contextual_orchestrator.nim_benchmark \ + -m pytest \ + tests/test_nim_benchmark.py \ + tests/test_nim_benchmark_release_acceptance.py \ + tests/test_nim_benchmark_workflow_contract.py \ + -q + python -m coverage report \ + --include=contextual_orchestrator/nim_benchmark.py \ + --show-missing \ + --fail-under=100 + python -m interrogate -f 100 contextual_orchestrator/nim_benchmark.py + + - name: Build, install, and import the wheel + run: | + set -euo pipefail + rm -rf dist "$RUNNER_TEMP/nim-wheel-site" + python -m pip wheel --no-deps --no-build-isolation . --wheel-dir dist + python -m pip install --no-deps \ + --target "$RUNNER_TEMP/nim-wheel-site" \ + dist/contextual_orchestrator-*.whl + cd "$RUNNER_TEMP" + PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \ + python -c "import contextual_orchestrator; import contextual_orchestrator.nim_benchmark" diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index bf53e3e5d..efe05d083 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -93,6 +93,9 @@ jobs: - name: Fuzz reasoning-effort profile parser run: python fuzz/fuzz_reasoning_effort_profile.py -max_total_time="${FUZZ_SECONDS}" -artifact_prefix=crash- fuzz/corpus/reasoning_effort_profile + - name: Fuzz NIM model-catalog parser + run: python fuzz/fuzz_nim_catalog.py -max_total_time="${FUZZ_SECONDS}" -artifact_prefix=crash- fuzz/corpus/nim_catalog + - name: Upload crash artifacts if: failure() uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5 diff --git a/.github/workflows/nim-benchmark.yml b/.github/workflows/nim-benchmark.yml new file mode 100644 index 000000000..fb645649d --- /dev/null +++ b/.github/workflows/nim-benchmark.yml @@ -0,0 +1,151 @@ +name: NIM benchmark + +# Evidence-grade NVIDIA NIM model discovery and cost-quality benchmark. +# Manual dispatch defaults to a deterministic dry run. Monthly scheduled runs +# are live but use a conservative hard request cap. The first-of-month schedule +# keeps the next run inside the current reviewed evidence window; stale evidence +# still fails closed. Dry execution receives no provider credential; only the +# live step can read NVIDIA_NIM_API_KEY. + +on: + workflow_dispatch: + inputs: + dry_run: + description: "Dry run without contacting NVIDIA" + type: boolean + default: true + max_total_requests: + description: "Hard cap on provider requests for this run" + type: number + default: 2000 + pricing_scenario: + description: "Optional reviewed pricing-scenario JSON path" + type: string + default: "" + schedule: + - cron: "23 3 1 * *" + +permissions: + contents: read + +concurrency: + group: nim-benchmark + cancel-in-progress: false + +jobs: + dry_run_benchmark: + name: Deterministic NIM benchmark dry run + if: github.event_name == 'workflow_dispatch' && inputs.dry_run == true + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install pinned runtime + run: | + python -m pip install --require-hashes -r requirements.lock + python -m pip install --no-deps -e . + + - name: Run dry benchmark + env: + MAX_REQUESTS: ${{ inputs.max_total_requests }} + PRICING_SCENARIO: ${{ inputs.pricing_scenario }} + PROVENANCE_GIT_SHA: ${{ github.sha }} + PROVENANCE_RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + extra_args=() + if [ -n "$PRICING_SCENARIO" ]; then + extra_args+=(--pricing-scenario "$PRICING_SCENARIO") + fi + python -m contextual_orchestrator nim-benchmark \ + --dry-run \ + "${extra_args[@]}" \ + --task-manifest examples/nim_task_manifest.json \ + --output-dir benchmark_artifacts \ + --max-total-requests "$MAX_REQUESTS" \ + --git-sha "$PROVENANCE_GIT_SHA" \ + --workflow-run-id "$PROVENANCE_RUN_ID" + + - name: Upload dry-run artifacts + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5 + with: + name: nim-benchmark-dry-${{ github.run_id }} + path: benchmark_artifacts/ + retention-days: 30 + if-no-files-found: error + + live_benchmark: + name: Live NIM catalog benchmark + if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.dry_run != true) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install pinned runtime + run: | + python -m pip install --require-hashes -r requirements.lock + python -m pip install --no-deps -e . + + - name: Resolve live parameters + id: live_params + env: + EVENT_NAME: ${{ github.event_name }} + INPUT_MAX_REQUESTS: ${{ inputs.max_total_requests }} + INPUT_PRICING: ${{ inputs.pricing_scenario }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" = "schedule" ]; then + echo "max_requests=2000" >> "$GITHUB_OUTPUT" + echo "pricing_scenario=" >> "$GITHUB_OUTPUT" + else + echo "max_requests=${INPUT_MAX_REQUESTS}" >> "$GITHUB_OUTPUT" + echo "pricing_scenario=${INPUT_PRICING}" >> "$GITHUB_OUTPUT" + fi + + - name: Run live benchmark + env: + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + MAX_REQUESTS: ${{ steps.live_params.outputs.max_requests }} + PRICING_SCENARIO: ${{ steps.live_params.outputs.pricing_scenario }} + PROVENANCE_GIT_SHA: ${{ github.sha }} + PROVENANCE_RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + extra_args=() + if [ -n "$PRICING_SCENARIO" ]; then + extra_args+=(--pricing-scenario "$PRICING_SCENARIO") + fi + python -m contextual_orchestrator nim-benchmark \ + "${extra_args[@]}" \ + --task-manifest examples/nim_task_manifest.json \ + --output-dir benchmark_artifacts \ + --max-total-requests "$MAX_REQUESTS" \ + --git-sha "$PROVENANCE_GIT_SHA" \ + --workflow-run-id "$PROVENANCE_RUN_ID" + + - name: Upload live benchmark artifacts + if: always() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5 + with: + name: nim-benchmark-live-${{ github.run_id }} + path: benchmark_artifacts/ + retention-days: 90 + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index afc68614e..b28758e19 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ tempcred.txt .hypothesis/ .secrets/ .coverage + +# local benchmark artifacts (uploaded by CI, not committed) +benchmark_artifacts/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b38e0770..5895f68b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- NVIDIA NIM benchmark cells now reconcile reported prompt and completion + usage independently, preventing negative completion counts after failures. - OpenRouter discovery no longer marks the entire credential account evidence-only. Authenticated catalog rows may serve ordinary requests, while ZDR-only requests still require explicit route-level ZDR evidence. @@ -114,12 +116,19 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) `role_effort_catalog=default_role_effort_catalog()` to attach the same `reasoning_effort_snapshot` on `complete`, `run`, `stream_route`, and `batch_route`; omit it to keep today's payload. +- Add an optional provider-neutral NVIDIA NIM benchmark harness that dynamically discovers the live `/v1/models` catalog, probes every discovered model under bounded concurrency and a hard request cap, records machine-readable capability outcomes, and compares direct, route-once, bounded-conduct, and explicit pricing-scenario policies over a locked task manifest. +- Add deterministic no-egress benchmark dry runs, secret-redacted JSON/CSV/Markdown evidence artifacts, paired bootstrap uncertainty, quality-latency and quality-hypothetical-cost Pareto frontiers, all-modality catalog fuzzing, and a manually gated benchmark workflow. +- Add a validated deterministic one-frame H.264 MP4 probe fixture, complete preflight reservation for every discovered model-capability cell plus the full evaluation envelope, and explicit evidence-sufficiency fields that keep the bundled smoke manifest from authorizing production routing. +- Add direct benchmark quality gates for 100% production statement/branch coverage, 100% public docstrings, wheel build/install/import smoke testing, and optional-import isolation. - Streamed `/v1/responses` workflow runs now request provider usage only from agents explicitly marked `stream_usage_supported`, preserve provider-declared SSE usage, record per-step `stream` cost-ledger rows, and expose cost status plus usage-record identities. Missing provider usage is explicitly unavailable; the gateway does not estimate billing tokens from the final answer, and nested gateway upstreams remain compatible (ADR 0040). +- Experimental CEFR criterion-observation gateway with exact contract checks, + independent rater blindness, bounded structured-output parsing, replay + provenance, and human-review routing; it emits no final CEFR level or score. ### Fixed @@ -338,6 +347,31 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) - Strix B105 false positives eliminated at the source: KV credential-name constants renamed `*_CREDENTIAL_NAME`; readiness label keys renamed `readiness_ok/warning/failure`. (#833) +- Expose a stable complete-run request planning view and align API, CLI, manual-workflow, and deterministic test caps with the locked thirty-task evidence floor, preserving fail-before-probe behavior. +- Fail closed after catalog discovery but before capability egress when the complete all-model probe and equal-budget evaluation plan cannot fit the configured hard request cap; the monthly 2,000-request ceiling covers the representative 127-model, thirty-task, seven-worker plan requiring 1,924 requests, including route-once's full equal-call envelope and direct-cell judge calls, and still rejects larger plans before partial probing. +- Align the monthly NIM live schedule with the reviewed access-cost evidence window while preserving fail-closed behavior after its validity horizon. +- Scale the equal NIM policy-cell token budget with the five-call envelope so the conduct arm can carry its prompts while every policy retains the same total allowance. +- Treat provider HTTP 401 and 403 responses during capability probes as authentication rejection, and keep live evaluation on the same DNS-pinned benchmark transport used by discovery and probes. +- Pin Atheris by Python interpreter so the Python 3.11 fuzz job and the newer central coverage-evidence image both install a published, hash-locked wheel. +- Record the reviewed current NVIDIA NIM General FAQ as expiring evidence for free Developer Program hosted-endpoint prototyping access, while keeping NVIDIA AI Enterprise production licensing and every hypothetical model rate explicitly separate. +- Require live hypothetical pricing scenarios to carry reviewed source, reviewer, review date, validity horizon, rate basis, uncertainty, and explicit rates; reject unreviewed, incomplete, future-dated, or expired price evidence before provider egress. +- Give direct, route-once, conduct, and reviewed cheapest-worker cells one equal total prompt-plus-completion token budget and one common five-call envelope, with configured-versus-observed evidence in every cell. +- Keep the optional NIM adapter lazy: importing the runtime package no longer imports the benchmark or mutates benchmark globals. +- Record immutable source-artifact digests and exact Git tree identity in the integration evidence so buyers and reviewers can reproduce the accepted benchmark source independently of transient workflow state. + +- NIM benchmark provider responses are bounded to 8 MiB, and live HTTPS + requests use validation-time public-address pinning with original-host TLS, + no proxy lookup, and no redirect following. +- Live pricing evidence is rejected unless its source, reviewer, dates, rate + basis, uncertainty, and explicit rates are complete and current. +- Direct, route-once, conduct, and reviewed cheapest-worker cells share one + total token budget and five-call envelope, with configured and observed + values recorded separately. +- Complete catalog probing and the full evaluation reserve are planned before + capability egress; the benchmark fails closed when the configured cap is too + small, and the scheduled workflow uses a reviewed 2,000-request ceiling. +- The NIM access-cost evidence, hypothetical pricing provenance, and source + artifact digests remain explicit and independently reproducible. ### Security @@ -349,6 +383,9 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) - Worker-agent pool boundaries are enforced beside object lookup so a different-pool id can no longer read or mutate another pool's agent. +- Provider hosts resolving to any non-globally-routable address are rejected, + including RFC 6598 shared space; benchmark artifacts refuse secret leakage. + ### References - Sakana AI. (2026). *Sakana Fugu Technical Report*. diff --git a/README.md b/README.md index 60ffff831..08eaca830 100644 --- a/README.md +++ b/README.md @@ -296,9 +296,35 @@ is read from a **KV config store**, never `os.getenv`. `pg_tiktoken` counting, and the production batch backend without adding a repository split here. -Grounding papers (LLM cost, routing, load balancing) live in +Grounding papers (LLM cost, routing, load balancing, evaluation) live in [docs/papers](docs/papers/README.md) with citations. +### NIM cost-quality benchmark (optional harness) + +Evidence-grade benchmark of the routing policies against a **dynamically +discovered** NVIDIA NIM catalog. It probes chat, completions, Responses, +embeddings, image/video/audio understanding, transcription, and speech; compares +direct, route-once, and bounded-conduct cells under one equal total-token and +call budget; records paired uncertainty and Pareto frontiers; and keeps reviewed +actual endpoint-access evidence separate from optional hypothetical paid rates. +The bundled manifest is smoke-sized and reports `evidence_review_required` only +after its paired cells complete; `routing_recommendation` remains null and no +benchmark artifact automatically changes production routing. + +The adapter is lazy and optional: ordinary `import contextual_orchestrator` does +not import or mutate it. Deterministic `--dry-run` receives no network access or +NVIDIA secret. Live execution resolves `NVIDIA_NIM_API_KEY` from the credential +registry, pins HTTPS connections to validation-time public addresses, rejects +redirects and proxy routing, and fails closed on missing/expired evidence. See +[docs/nim_benchmark.md](docs/nim_benchmark.md) and the +[engineering decision record](docs/doctoring/nim-benchmark-evidence-grade.md). + +```bash +python -m contextual_orchestrator nim-benchmark --dry-run \ + --pricing-scenario examples/nim_pricing_scenario.json \ + --output-dir benchmark_artifacts +``` + ## Design Artifacts - [Library research](docs/library_research.md) @@ -363,6 +389,7 @@ python -m pytest -q tests/test_reasoning_effort_profile.py python tests/test_admin_contract.py python tests/test_conventions.py python tests/test_api_contract.py +python tests/test_nim_benchmark.py python tests/test_security_hardening.py python tests/test_chat_model_capability_isolation.py python tests/test_chat_transport_role_separation.py diff --git a/conductor/tracks.md b/conductor/tracks.md index b377ec28c..6f1f9ec5b 100644 --- a/conductor/tracks.md +++ b/conductor/tracks.md @@ -5,3 +5,4 @@ | 001-paper-grounded-orchestrator | active | Implement the source-backed orchestration contract with TDD, DDD, and CDD | | 002-enterprise-design-foundation | active | Add paper-grounded screen design, user stories, REST API, code/DB conventions, and i18n | | 003-reasoning-effort-profiles | active | Issue #568: versioned per-role `reasoning_effort_profile`, equal-budget θ̂ RMSE ablation, snapshot on run/stream/batch, production defaults locked | +| 004-nim-cost-quality-benchmark | active | Evidence-grade NIM catalog discovery, all-modality capability probes, and the route/conduct/single-worker cost-quality benchmark (docs/nim_benchmark.md) | diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index ffd82536d..5d739c3eb 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -494,6 +494,13 @@ def main(argv: list[str] | None = None) -> None: _check_fast_mlsirm_command() return + if arguments and arguments[0] == "nim-benchmark": + # Optional benchmark harness (issue #86): dynamic NIM catalog discovery, + # all-modality capability probes, and the cost-quality policy benchmark. + from .nim_benchmark import run_benchmark_cli + + sys.exit(run_benchmark_cli(arguments[1:])) + parser = argparse.ArgumentParser(description="Route or conduct chat requests across model agents.") parser.add_argument("prompt", nargs="?", help="User prompt for CLI mode.") parser.add_argument("--agents", default="examples/agents.mock.json", help="Agent config JSON.") diff --git a/contextual_orchestrator/nim_benchmark.py b/contextual_orchestrator/nim_benchmark.py new file mode 100644 index 000000000..3e20daf95 --- /dev/null +++ b/contextual_orchestrator/nim_benchmark.py @@ -0,0 +1,3427 @@ +"""Evidence-grade NVIDIA NIM model discovery and cost-quality benchmark harness. + +This is the optional benchmark adapter demanded by the "[Product Gap] +Evidence-grade NVIDIA NIM model discovery and cost-quality benchmark" issue. +It is NOT part of the runtime request path: the gateway keeps its +provider-neutral, standard-library-only contract, and this module simply +reuses the same stdlib HTTP/KV seams to measure the repo's own policies +(``route_once`` vs ``conduct`` vs single-worker baselines) against a +dynamically discovered NIM catalog. + +Design contract (mirrors the issue): + +* **Dynamic catalog** — models come from the OpenAI-compatible + ``GET /v1/models`` endpoint; nothing here hard-codes a model inventory. +* **All-modality capability probes** — every discovered model is probed, + under bounded concurrency and a hard request budget, for every contract + NIM can host: chat completions, legacy text completions, the Responses + API, embeddings, image understanding (vision), video understanding, + audio understanding (omni-style ``input_audio``), audio transcription, + and audio speech synthesis. ``omni_capable`` is derived, never probed + separately. A run that cannot execute every cell fails before capability egress. +* **Fair comparison** — the same task manifest, scorers, call caps, + workflow-depth cap (five), timeout, and output-token budget apply to all + compared systems. +* **Honest cost accounting** — actual cost is recorded as ``0`` while the + hosted catalog is free to the caller; hypothetical paid cost is computed + only from an explicit versioned pricing scenario and is ``"unknown"`` + for any model the scenario does not price. The two never mix. +* **Fail closed** — a live run refuses to start without the KV-resolvable + ``NVIDIA_NIM_API_KEY`` credential, complete provenance, and a request + budget large enough for the planned evaluation. The secret is never + accepted via argv and never serialized into artifacts. +* **Deterministic dry run** — ``--dry-run`` drives the entire pipeline + against an in-process synthetic provider covering every modality class, + so manifests, pricing assumptions, scorer registration, budgets, and + output schemas are validated without any network egress. +""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import datetime as datetime_module +import decimal +import hashlib +import http.client +import io +import json +import math +import os +import random +import re +import socket +import ssl +import struct +import threading +import time +import urllib.error +import urllib.parse +import wave +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Callable + +from .conventions import is_two_word_snake_case +from .credentials import NotConfigured, get_credential, register_credential +from .nim_evidence import publish_artifact_set +from .orchestrator import ( + ModelAgent, + ModelClient, + OrchestrationPolicy, + ReasoningEffortProfile, + TaskOrchestrator, + estimate_tokens, + redact_text, +) +from .provider_transport import ( + PinnedHTTPSConnection, + validated_public_addresses, +) + +BENCHMARK_SCHEMA_VERSION = "1.0.0" +NIM_DEFAULT_ENDPOINT = "https://integrate.api.nvidia.com/v1" +NIM_CREDENTIAL_NAME = "NVIDIA_NIM_API_KEY" +DRY_RUN_PROVENANCE_PLACEHOLDER = "dry_run" +# Fixed epoch for deterministic dry-run artifacts (2026-01-01T00:00:00Z). +DRY_RUN_FIXED_UNIX_TIME = 1767225600.0 +# Issue contract: Conductor/TRINITY-style deep paths are capped at five steps. +MAX_WORKFLOW_DEPTH = 5 +# Provider output remains capped at 264 tokens by default. The equal cell-wide +# prompt-plus-completion budget scales with the maximum five-call envelope so a +# fixed conduct workflow can carry its prompts without being starved. The +# eight-token margin over the historical 256 keeps the locked 30-task +# manifest's tightest conduct_bounded task (four-call accumulated prompt +# context) inside its equal budget under the current deterministic dry-run +# token estimate; see test_smoke_manifest_cannot_authorize_production_routing. +DEFAULT_MAX_OUTPUT_TOKENS = 264 +DEFAULT_POLICY_TOTAL_TOKEN_BUDGET = MAX_WORKFLOW_DEPTH * DEFAULT_MAX_OUTPUT_TOKENS +# Bound every provider response before materializing it in memory. Eight MiB is +# ample for model catalogs, JSON probe responses, and the deliberately tiny +# benchmark media outputs while preventing a provider from returning an +# unbounded body to the evidence collector. +MAX_PROVIDER_RESPONSE_BYTES = 8 * 1024 * 1024 +# Smoke manifests can exercise plumbing but cannot justify production routing. +MINIMUM_PAIRED_TASK_COUNT = 30 +REQUIRED_COMPLETION_FRACTION = 0.9 + +ACTUAL_COST_EVIDENCE: dict[str, Any] = { + "evidence_schema_version": "1.0.0", + "source_title": "NVIDIA NIM General FAQ", + "source_url": "https://docs.api.nvidia.com/nim/docs/product", + "reviewed_at_date": "2026-08-05", + "valid_until_date": "2026-09-04", + "access_program": "NVIDIA Developer Program API Catalog hosted endpoints", + "access_scope": "free API endpoint access for prototyping", + "production_access_note": ( + "Production support and licensing require NVIDIA AI Enterprise." + ), + "actual_cost_usd": 0.0, + "uncertainty": ( + "Hosted-endpoint access terms can change. Live runs fail closed after " + "the validity date until the official source is reviewed again." + ), +} + +# Transport seam: (method, url, headers, body_bytes_or_None) -> (status, body). +# Network-level failures raise URLError/TimeoutError/ConnectionError/socket.timeout. +ProviderTransport = Callable[ + [str, str, dict[str, str], bytes | None], tuple[int, bytes] +] + + +class BenchmarkContractError(ValueError): + """A manifest, pricing scenario, schema, or parameter violates the benchmark contract.""" + + +class CatalogDiscoveryError(RuntimeError): + """The provider model catalog could not be discovered or parsed completely.""" + + +class BenchmarkAuthError(RuntimeError): + """The provider rejected the benchmark credential; the run must fail closed.""" + + +class BenchmarkBudgetError(RuntimeError): + """The benchmark would exceed (or has exceeded) its hard request budget.""" + + +class SecretLeakError(RuntimeError): + """A serialized artifact contained the provider secret; writing is refused.""" + + +# -------------------------------------------------------------------------- +# Egress guard + default transport +# -------------------------------------------------------------------------- + + +def require_public_https_endpoint(url: str) -> tuple[str, ...]: + """Resolve and return public addresses approved for one HTTPS request. + + The returned addresses are the only addresses a caller may dial. Combining + resolution and validation in one operation closes the DNS time-of-check to + time-of-use gap caused by a generic URL opener resolving the hostname again. + + Args: + url: Complete provider URL whose origin may receive credentials. + + Returns: + Deduplicated globally routable IPv4 or IPv6 addresses. + + Raises: + BenchmarkContractError: If the URL is not HTTPS, lacks a hostname, or + resolves to any non-global address. + """ + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "https" or not parsed.hostname: + raise BenchmarkContractError(f"benchmark endpoint must use https: {url!r}") + try: + return validated_public_addresses( + parsed.hostname.lower(), + parsed.port or 443, + "NIM benchmark", + ) + except RuntimeError as exc: + raise BenchmarkContractError(str(exc)) from exc + + +def build_default_transport(timeout_seconds: float) -> ProviderTransport: + """Build direct HTTPS transport pinned to each request's DNS evidence. + + Every request resolves exactly once, validates every answer as globally + routable, and connects only to those validation-time addresses. The original + hostname remains the HTTP authority and TLS SNI/certificate name. Environment + proxies and redirect handlers are never used. + + Args: + timeout_seconds: Socket, TLS, and response timeout for each address. + + Returns: + A provider transport returning HTTP status and raw response bytes. + + Raises: + BenchmarkContractError: If a request URL or redirect violates policy. + urllib.error.URLError: If all validation-time addresses fail. + """ + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, (int, float)) + or not math.isfinite(timeout_seconds) + or timeout_seconds <= 0 + ): + raise BenchmarkContractError("timeout_seconds must be a positive number") + ssl_context = ssl.create_default_context() + + def transport( + method: str, + url: str, + headers: dict[str, str], + body: bytes | None, + ) -> tuple[int, bytes]: + """Perform one request without proxy lookup, redirect follow, or re-resolution.""" + parsed = urllib.parse.urlparse(url) + approved_addresses = require_public_https_endpoint(url) + port = parsed.port or 443 + target = parsed.path or "/" + if parsed.params: + target = f"{target};{parsed.params}" + if parsed.query: + target = f"{target}?{parsed.query}" + request_headers = dict(headers) + request_headers["Connection"] = "close" + + last_error: BaseException | None = None + for pinned_ip in approved_addresses: + connection = PinnedHTTPSConnection( + parsed.hostname or "", + pinned_ip, + port, + float(timeout_seconds), + ssl_context, + ) + response = None + try: + connection.request( + method, + target, + body=body, + headers=request_headers, + ) + response = connection.getresponse() + status = int(response.status) + response_body = response.read(MAX_PROVIDER_RESPONSE_BYTES + 1) + if len(response_body) > MAX_PROVIDER_RESPONSE_BYTES: + raise BenchmarkContractError( + "benchmark provider response exceeds " + f"{MAX_PROVIDER_RESPONSE_BYTES} byte limit" + ) + if 300 <= status < 400: + raise BenchmarkContractError( + f"benchmark provider redirects are not permitted (HTTP {status})" + ) + return status, response_body + except BenchmarkContractError: + raise + except (OSError, http.client.HTTPException) as exc: + last_error = exc + finally: + if response is not None: + response.close() + connection.close() + raise urllib.error.URLError( + last_error or "benchmark provider connection failed" + ) + + return transport + + +# -------------------------------------------------------------------------- +# Request budget (fail-closed hard cap) +# -------------------------------------------------------------------------- + + +class RequestBudget: + """Thread-safe hard cap on total provider requests for one benchmark run.""" + + def __init__(self, max_total_requests: int) -> None: + """Create a positive integer request allowance. + + Args: + max_total_requests: Maximum provider calls in the complete run. + + Raises: + BenchmarkContractError: If the cap is boolean or not positive. + """ + if ( + isinstance(max_total_requests, bool) + or not isinstance(max_total_requests, int) + or max_total_requests < 1 + ): + raise BenchmarkContractError( + "max_total_requests must be a positive integer" + ) + self.max_total_requests = max_total_requests + self._spent = 0 + self._lock = threading.Lock() + + def try_spend(self) -> bool: + """Consume one request from the budget; return ``False`` when exhausted.""" + with self._lock: + if self._spent >= self.max_total_requests: + return False + self._spent += 1 + return True + + def spend_or_fail(self) -> None: + """Consume one request or raise for a phase that must complete.""" + if not self.try_spend(): + raise BenchmarkBudgetError( + f"request budget of {self.max_total_requests} exhausted; " + "refusing further provider calls" + ) + + @property + def requests_spent(self) -> int: + """Return the number of provider requests consumed so far.""" + with self._lock: + return self._spent + + @property + def remaining_requests(self) -> int: + """Return the non-negative provider request allowance still available.""" + with self._lock: + return self.max_total_requests - self._spent + + +class _BudgetedModelClient(ModelClient): + """ModelClient that charges every chat call against the shared request budget.""" + + def __init__( + self, + request_budget: RequestBudget, + transport: ProviderTransport | None = None, + **kwargs: Any, + ) -> None: + # ponytail: disable hidden provider retries so the hard request budget + # bounds actual egress rather than only logical chat calls. + kwargs["max_retries"] = 0 + super().__init__(**kwargs) + self._request_budget = request_budget + self._benchmark_transport = transport + self._benchmark_contract_error: BenchmarkContractError | None = None + + @property + def benchmark_contract_error(self) -> BenchmarkContractError | None: + """Return the first benchmark transport-contract failure, if any.""" + return self._benchmark_contract_error + + def _send( + self, + agent: ModelAgent, + payload: dict[str, Any], + destination: Any = None, + *, + timeout: float | None = None, + ) -> str: + """Send evaluation chat calls through the benchmark's pinned transport.""" + if self._benchmark_transport is None or agent.base_url.startswith("mock://"): + return super()._send(agent, payload, destination, timeout=timeout) + url = self._provider_url(agent, "/chat/completions") + try: + status, body = self._benchmark_transport( + "POST", + url, + _auth_headers(get_credential(NIM_CREDENTIAL_NAME) or ""), + json.dumps(payload).encode("utf-8"), + ) + except BenchmarkContractError as exc: + self._benchmark_contract_error = exc + raise + if status in (401, 403): + raise BenchmarkAuthError( + f"provider rejected the benchmark credential during evaluation (HTTP {status})" + ) + if status >= 400: + raise urllib.error.HTTPError( + url, + status, + "NIM benchmark provider request failed", + {}, + io.BytesIO(body), + ) + try: + data = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as exc: + contract_error = BenchmarkContractError( + "evaluation provider response must be valid JSON" + ) + self._benchmark_contract_error = contract_error + raise contract_error from exc + if not isinstance(data, dict): + contract_error = BenchmarkContractError( + "evaluation provider response must be an object" + ) + self._benchmark_contract_error = contract_error + raise contract_error + usage = data.get("usage") + if isinstance(usage, dict): + self._local.usage = usage + return self._response_content(agent, data) + + def chat( + self, + agent: ModelAgent, + messages: list[dict[str, str]], + temperature: float | None = None, + top_p: float | None = None, + effort_profile: ReasoningEffortProfile | None = None, + ) -> str: + """Spend one budgeted request, then delegate to the normal chat path.""" + self._request_budget.spend_or_fail() + return super().chat(agent, messages, temperature, top_p, effort_profile) + + def proxy_send( + self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] + ) -> dict[str, Any]: + """Charge and send one structured evaluation request.""" + self._request_budget.spend_or_fail() + if self._benchmark_transport is None or agent.base_url.startswith("mock://"): + return super().proxy_send_once(agent, endpoint, payload) + url = self._provider_url(agent, f"/{endpoint.strip('/')}") + try: + status, body = self._benchmark_transport( + "POST", + url, + _auth_headers(get_credential(NIM_CREDENTIAL_NAME) or ""), + json.dumps(payload).encode("utf-8"), + ) + except BenchmarkContractError as exc: + self._benchmark_contract_error = exc + raise + if status in (401, 403): + raise BenchmarkAuthError( + f"provider rejected the benchmark credential during evaluation (HTTP {status})" + ) + if status >= 400: + raise urllib.error.HTTPError( + url, + status, + "NIM benchmark provider request failed", + {}, + io.BytesIO(body), + ) + try: + response = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as exc: + contract_error = BenchmarkContractError( + "structured provider response must be valid JSON" + ) + self._benchmark_contract_error = contract_error + raise contract_error from exc + if not isinstance(response, dict): + contract_error = BenchmarkContractError( + "structured provider response must be an object" + ) + self._benchmark_contract_error = contract_error + raise contract_error + return response + + def proxy_send_once( + self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] + ) -> dict[str, Any]: + """Use the same single-attempt budget boundary for endpoint races.""" + return self.proxy_send(agent, endpoint, payload) + + +class PolicyTokenBudgetExceeded(RuntimeError): + """A policy cell exhausted its shared token or call allowance.""" + + +class EqualBudgetModelClient: + """Delegate model calls while enforcing an equal per-cell budget. + + Direct, route-once, conduct, and cheapest-worker cells all receive the same + total prompt-plus-completion token allowance and the same declared maximum- + call envelope. The wrapper lowers each provider call's output cap to the + remaining allowance and reconciles estimates with provider-reported usage. + """ + + def __init__( + self, + delegate: ModelClient, + total_token_budget: int, + maximum_calls: int, + ) -> None: + """Create a cell-local limiter around an existing provider client. + + Args: + delegate: Existing request-budgeted provider client. + total_token_budget: Cell-wide prompt-plus-completion allowance. + maximum_calls: Maximum calls available to every compared policy. + + Raises: + ValueError: If either allowance is boolean or not positive. + """ + if ( + isinstance(total_token_budget, bool) + or not isinstance(total_token_budget, int) + or total_token_budget < 1 + ): + raise ValueError("total_token_budget must be a positive integer") + if ( + isinstance(maximum_calls, bool) + or not isinstance(maximum_calls, int) + or maximum_calls < 1 + ): + raise ValueError("maximum_calls must be a positive integer") + self._delegate = delegate + self.total_token_budget = total_token_budget + self.maximum_calls = maximum_calls + self.observed_calls = 0 + self.observed_tokens = 0 + self.observed_prompt_tokens = 0 + self.observed_completion_tokens = 0 + self.attempted_models: list[dict[str, Any]] = [] + self.estimated_usage_by_model: dict[str, dict[str, int]] = {} + self._pending_estimated_usage: tuple[str, int, int] | None = None + self._exceeded = False + self._contract_error: BenchmarkContractError | None = None + + def __getattr__(self, name: str) -> Any: + """Forward provider-client capabilities not owned by the cell limiter.""" + return getattr(self._delegate, name) + + @property + def max_output_tokens(self) -> int: + """Expose the delegate cap for compatibility with orchestration clients.""" + return int(self._delegate.max_output_tokens) + + @max_output_tokens.setter + def max_output_tokens(self, value: int) -> None: + """Forward explicit cap changes to the delegated model client.""" + self._delegate.max_output_tokens = value + + @property + def remaining_tokens(self) -> int: + """Return the non-negative token allowance remaining in this cell.""" + return max(0, self.total_token_budget - self.observed_tokens) + + @property + def exceeded(self) -> bool: + """Return whether observed usage crossed the configured allowance.""" + return self._exceeded + + @property + def contract_error(self) -> BenchmarkContractError | None: + """Return a transport-contract failure swallowed by orchestration failover.""" + return self._contract_error + + @staticmethod + def _coerce_usage_count(value: Any) -> int | None: + """Return one valid non-negative provider token count, otherwise ``None``.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + if not math.isfinite(value) or value < 0: + return None + return int(value) + + def chat( + self, + agent: ModelAgent, + messages: list[dict[str, Any]], + temperature: float | None = None, + top_p: float | None = None, + effort_profile: ReasoningEffortProfile | None = None, + ) -> str: + """Perform one delegated call within the remaining cell allowance. + + Raises: + PolicyTokenBudgetExceeded: If the call or token allowance is already + exhausted or the prompt cannot fit. + """ + if self._exceeded or self.observed_calls >= self.maximum_calls: + raise PolicyTokenBudgetExceeded( + "policy cell maximum-call allowance exhausted" + ) + prompt_text = json.dumps(messages, ensure_ascii=False, sort_keys=True) + prompt_tokens = estimate_tokens(prompt_text) + output_allowance = self.remaining_tokens - prompt_tokens + if output_allowance < 1: + raise PolicyTokenBudgetExceeded( + "policy cell total-token allowance exhausted" + ) + + output_cap = min(int(self._delegate.max_output_tokens), output_allowance) + self.observed_calls += 1 + self.observed_prompt_tokens += prompt_tokens + self.observed_tokens += prompt_tokens + self.attempted_models.append( + {"role": "attempted", "agent_id": agent.id, "model_id": agent.model} + ) + usage = self.estimated_usage_by_model.setdefault( + agent.model, {"prompt_tokens": 0, "completion_tokens": 0} + ) + usage["prompt_tokens"] += prompt_tokens + try: + with self._delegate.request_settings(max_output_tokens=output_cap): + answer = self._delegate.chat( + agent, + messages, + temperature, + top_p, + effort_profile, + ) + finally: + delegate_error = getattr(self._delegate, "benchmark_contract_error", None) + if isinstance(delegate_error, BenchmarkContractError): + self._contract_error = delegate_error + + completion_tokens = estimate_tokens(answer) + self.observed_tokens += completion_tokens + self.observed_completion_tokens += completion_tokens + usage["completion_tokens"] += completion_tokens + self._pending_estimated_usage = (agent.model, prompt_tokens, completion_tokens) + self._exceeded = self.observed_tokens > self.total_token_budget + return answer + + def proxy_send( + self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] + ) -> dict[str, Any]: + """Apply the cell call/token envelope to structured judge requests.""" + if self._exceeded or self.observed_calls >= self.maximum_calls: + raise PolicyTokenBudgetExceeded( + "policy cell maximum-call allowance exhausted" + ) + prompt_tokens = estimate_tokens( + json.dumps(payload, ensure_ascii=False, sort_keys=True) + ) + output_allowance = self.remaining_tokens - prompt_tokens + if output_allowance < 1: + raise PolicyTokenBudgetExceeded( + "policy cell total-token allowance exhausted" + ) + request = dict(payload) + requested_cap = request.get("max_tokens") + request["max_tokens"] = min( + requested_cap if type(requested_cap) is int and requested_cap > 0 else output_allowance, + output_allowance, + ) + self.observed_calls += 1 + self.observed_prompt_tokens += prompt_tokens + self.observed_tokens += prompt_tokens + self.attempted_models.append( + {"role": "attempted", "agent_id": agent.id, "model_id": agent.model} + ) + model_usage = self.estimated_usage_by_model.setdefault( + agent.model, {"prompt_tokens": 0, "completion_tokens": 0} + ) + model_usage["prompt_tokens"] += prompt_tokens + response = self._delegate.proxy_send(agent, endpoint, request) + answer = ModelClient._response_content(agent, response) + completion_tokens = estimate_tokens(answer) + self.observed_tokens += completion_tokens + self.observed_completion_tokens += completion_tokens + model_usage["completion_tokens"] += completion_tokens + usage = response.get("usage") + if isinstance(usage, dict): + reported_prompt = self._coerce_usage_count(usage.get("prompt_tokens")) + reported_completion = self._coerce_usage_count(usage.get("completion_tokens")) + if reported_prompt is not None and reported_completion is not None: + self.observed_prompt_tokens += reported_prompt - prompt_tokens + self.observed_completion_tokens += reported_completion - completion_tokens + self.observed_tokens += ( + reported_prompt + reported_completion - prompt_tokens - completion_tokens + ) + model_usage["prompt_tokens"] += reported_prompt - prompt_tokens + model_usage["completion_tokens"] += reported_completion - completion_tokens + self._exceeded = self.observed_tokens > self.total_token_budget + return response + + def proxy_send_once( + self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] + ) -> dict[str, Any]: + """Keep endpoint-race structured sends inside the same cell boundary.""" + return self.proxy_send(agent, endpoint, payload) + + def take_usage(self) -> dict[str, Any] | None: + """Return delegated usage and replace the latest estimate when valid.""" + usage = self._delegate.take_usage() + pending = self._pending_estimated_usage + self._pending_estimated_usage = None + if pending is None or not isinstance(usage, dict): + return usage + prompt_tokens = self._coerce_usage_count(usage.get("prompt_tokens")) + completion_tokens = self._coerce_usage_count(usage.get("completion_tokens")) + if prompt_tokens is None or completion_tokens is None: + return usage + model, estimated_prompt, estimated_completion = pending + self.observed_prompt_tokens += prompt_tokens - estimated_prompt + self.observed_completion_tokens += completion_tokens - estimated_completion + self.observed_tokens = self.observed_prompt_tokens + self.observed_completion_tokens + model_usage = self.estimated_usage_by_model[model] + model_usage["prompt_tokens"] += prompt_tokens - estimated_prompt + model_usage["completion_tokens"] += completion_tokens - estimated_completion + self._exceeded = self.observed_tokens > self.total_token_budget + return usage + + +# -------------------------------------------------------------------------- +# Catalog discovery +# -------------------------------------------------------------------------- + + +def parse_model_catalog_body(body: bytes) -> dict[str, Any]: + """Parse an OpenAI-compatible ``GET /v1/models`` body into a hygienic inventory. + + Adversarial inputs (non-JSON, wrong shapes, entries without an id, + duplicate ids) never crash: structural failures raise + :class:`CatalogDiscoveryError`; salvageable per-entry problems are recorded + with machine-readable reasons in ``invalid_entries``/``duplicate_model_ids``. + """ + try: + decoded = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError, RecursionError) as exc: + raise CatalogDiscoveryError( + f"model catalog body is not valid JSON: {exc}" + ) from exc + if not isinstance(decoded, dict) or not isinstance(decoded.get("data"), list): + raise CatalogDiscoveryError( + "model catalog must be a JSON object with a 'data' list" + ) + + models: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + duplicate_model_ids: set[str] = set() + invalid_entries: list[dict[str, Any]] = [] + for index, entry in enumerate(decoded["data"]): + if not isinstance(entry, dict): + invalid_entries.append( + {"entry_index": index, "invalid_reason": "entry_not_an_object"} + ) + continue + model_id = entry.get("id") + if not isinstance(model_id, str) or not model_id.strip(): + invalid_entries.append( + {"entry_index": index, "invalid_reason": "missing_model_id"} + ) + continue + model_id = model_id.strip() + if model_id in seen_ids: + duplicate_model_ids.add(model_id) + continue + seen_ids.add(model_id) + owned_by = entry.get("owned_by") + models.append( + { + "model_id": model_id, + "owned_by": owned_by if isinstance(owned_by, str) else "", + } + ) + models.sort(key=lambda row: row["model_id"]) + return { + "models": models, + "duplicate_model_ids": sorted(duplicate_model_ids), + "invalid_entries": invalid_entries, + } + + +def discover_model_catalog( + transport: ProviderTransport, + endpoint: str, + api_key: str, + request_budget: RequestBudget, +) -> dict[str, Any]: + """Fetch and parse the live model catalog, failing closed on any discovery gap.""" + request_budget.spend_or_fail() + url = f"{endpoint.rstrip('/')}/models" + try: + status, body = transport("GET", url, _auth_headers(api_key), None) + except ( + urllib.error.URLError, + TimeoutError, + ConnectionError, + socket.timeout, + socket.gaierror, + ) as exc: + raise CatalogDiscoveryError( + f"model catalog request failed: {type(exc).__name__}" + ) from exc + if status in (401, 403): + raise BenchmarkAuthError( + f"provider rejected the benchmark credential (HTTP {status})" + ) + if status != 200: + raise CatalogDiscoveryError(f"model catalog request returned HTTP {status}") + catalog = parse_model_catalog_body(body) + if not catalog["models"]: + raise CatalogDiscoveryError( + "model catalog discovery returned zero usable models" + ) + return catalog + + +def _auth_headers( + api_key: str, content_type: str = "application/json" +) -> dict[str, str]: + """Standard provider headers; the bearer value never appears in artifacts.""" + return { + "authorization": f"Bearer {api_key}", + "content-type": content_type, + "accept": "application/json", + } + + +# -------------------------------------------------------------------------- +# Capability probes — every contract NIM can host +# -------------------------------------------------------------------------- + +# 1x1 transparent PNG for vision probes. +_TINY_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" +# Deterministic one-frame 16x16 H.264 MP4 generated once with bit-exact flags. +_TINY_MP4_BASE64 = """AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAALzbW9vdgAAAGxtdmhkAAAAAAAAAAAA +AAAAAAAD6AAAACgAAQAAAQAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAA +AABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAkJ0cmFrAAAAXHRraGQAAAADAAAA +AAAAAAAAAAABAAAAAAAAACgAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAA +AAAAAAAAAABAAAAAABAAAAAQAAAAAAAkZWR0cwAAABxlbHN0AAAAAAAAAAEAAAAoAAAAAAABAAAA +AAG6bWRpYQAAACBtZGhkAAAAAAAAAAAAAAAAAAAyAAAAAgBVxAAAAAAALWhkbHIAAAAAAAAAAHZp +ZGUAAAAAAAAAAAAAAABWaWRlb0hhbmRsZXIAAAABZW1pbmYAAAAUdm1oZAAAAAEAAAAAAAAAAAAA +ACRkaW5mAAAAHGRyZWYAAAAAAAAAAQAAAAx1cmwgAAAAAQAAASVzdGJsAAAAwXN0c2QAAAAAAAAA +AQAAALFhdmMxAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAABAAEABIAAAASAAAAAAAAAABDExhdmMg +bGlieDI2NAAAAAAAAAAAAAAAAAAAAAAAAAAAGP//AAAAN2F2Y0MBZAAK/+EAGWdkAAqscgRewEQA +AAMABAAAAwDIPEiWEYABAAdo6EOPEyEw/fj4AAAAABBwYXNwAAAAAQAAAAEAAAAUYnRydAAAAAAA +Ai3QAAAAAAAAABhzdHRzAAAAAAAAAAEAAAABAAACAAAAABxzdHNjAAAAAAAAAAEAAAABAAAAAQAA +AAEAAAAUc3RzegAAAAAAAALKAAAAAQAAABRzdGNvAAAAAAAAAAEAAAMjAAAAPXVkdGEAAAA1bWV0 +YQAAAAAAAAAhaGRscgAAAAAAAAAAbWRpcmFwcGwAAAAAAAAAAAAAAAAIaWxzdAAAAAhmcmVlAAAC +0m1kYXQAAAKyBgX//67cRem95tlIt5Ys2CDZI+7veDI2NCAtIGNvcmUgMTY0IHIzMTA4IDMxZTE5 +ZjkgLSBILjI2NC9NUEVHLTQgQVZDIGNvZGVjIC0gQ29weWxlZnQgMjAwMy0yMDIzIC0gaHR0cDov +L3d3dy52aWRlb2xhbi5vcmcveDI2NC5odG1sIC0gb3B0aW9uczogY2FiYWM9MSByZWY9MTYgZGVi +bG9jaz0xOi0zOi0zIGFuYWx5c2U9MHgzOjB4MTMzIG1lPXVtaCBzdWJtZT0xMCBwc3k9MSBwc3lf +cmQ9Mi4wMDowLjcwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTI0IGNocm9tYV9tZT0xIHRyZWxsaXM9 +MiA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29m +ZnNldD0tNCB0aHJlYWRzPTEgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5y +PTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2lu +dHJhPTAgYmZyYW1lcz04IGJfcHlyYW1pZD0yIGJfYWRhcHQ9MiBiX2JpYXM9MCBkaXJlY3Q9MyB3 +ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTI1IHNj +ZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NjAgcmM9Y3JmIG1idHJlZT0x +IGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0x +LjQwIGFxPTE6MS4yMACAAAAAEGWIgQAG5z/+9vD+BTZWBME=""" +VIDEO_PROBE_FIXTURE_SHA256 = ( + "777dda43b5a15162b68a39aa486d5c70c9994d7fe761742fd00d4e13508983c0" +) +_MULTIPART_BOUNDARY = "nim-benchmark-boundary-7f3a1c" +_MP4_CONTAINER_BOX_TYPES = frozenset( + {b"moov", b"trak", b"mdia", b"minf", b"dinf", b"stbl", b"edts", b"udta"} +) + + +def _iter_mp4_boxes( + data: bytes, + start_offset: int = 0, + end_offset: int | None = None, +): + """Yield validated ISO-BMFF boxes as type and payload/end offsets. + + Args: + data: Complete MP4 bytes. + start_offset: First byte of the bounded box sequence. + end_offset: Exclusive sequence end, defaulting to ``len(data)``. + + Yields: + Tuples of ``(box_type, payload_start, box_end)``. + + Raises: + BenchmarkContractError: If box headers, sizes, or bounds are malformed. + """ + sequence_end = len(data) if end_offset is None else end_offset + offset = start_offset + while offset < sequence_end: + if sequence_end - offset < 8: + raise BenchmarkContractError("video probe MP4 has a truncated box header") + box_size = struct.unpack(">I", data[offset : offset + 4])[0] + box_type = data[offset + 4 : offset + 8] + header_size = 8 + if box_size == 1: + if sequence_end - offset < 16: + raise BenchmarkContractError( + "video probe MP4 has a truncated extended box" + ) + box_size = struct.unpack(">Q", data[offset + 8 : offset + 16])[0] + header_size = 16 + elif box_size == 0: + box_size = sequence_end - offset + if box_size < header_size or offset + box_size > sequence_end: + raise BenchmarkContractError( + "video probe MP4 box exceeds its parent bounds" + ) + payload_start = offset + header_size + box_end = offset + box_size + yield box_type, payload_start, box_end + offset = box_end + + +def _walk_mp4_boxes(data: bytes): + """Yield every validated box in the deterministic video fixture.""" + + def walk(start_offset: int, end_offset: int): + """Recursively traverse known ISO-BMFF container boxes.""" + for box_type, payload_start, box_end in _iter_mp4_boxes( + data, + start_offset, + end_offset, + ): + yield box_type, payload_start, box_end + if box_type in _MP4_CONTAINER_BOX_TYPES: + yield from walk(payload_start, box_end) + elif box_type == b"meta": + if box_end - payload_start < 4: + raise BenchmarkContractError( + "video probe MP4 meta box lacks full-box flags" + ) + yield from walk(payload_start + 4, box_end) + + yield from walk(0, len(data)) + + +def validate_video_probe_fixture(data: bytes) -> dict[str, Any]: + """Validate one H.264 video stream, dimensions, and frame count. + + Args: + data: Candidate ISO-BMFF/MP4 bytes. + + Returns: + Codec, width, height, and frame count for the single video stream. + + Raises: + BenchmarkContractError: If required boxes or one-frame video evidence is + missing, inconsistent, or malformed. + """ + top_level_types = {box_type for box_type, _, _ in _iter_mp4_boxes(data)} + if not {b"ftyp", b"moov", b"mdat"} <= top_level_types: + raise BenchmarkContractError("video probe MP4 lacks ftyp, moov, or mdat") + + width: int | None = None + height: int | None = None + frame_count: int | None = None + video_handler_count = 0 + codec_name: str | None = None + for box_type, payload_start, box_end in _walk_mp4_boxes(data): + payload = data[payload_start:box_end] + if box_type == b"tkhd": + if len(payload) < 8: + raise BenchmarkContractError("video probe MP4 tkhd box is truncated") + width_fixed, height_fixed = struct.unpack(">II", payload[-8:]) + width = width_fixed >> 16 + height = height_fixed >> 16 + elif box_type == b"hdlr" and len(payload) >= 12: + if payload[8:12] == b"vide": + video_handler_count += 1 + elif box_type == b"stsz": + if len(payload) < 12: + raise BenchmarkContractError("video probe MP4 stsz box is truncated") + frame_count = struct.unpack(">I", payload[8:12])[0] + elif box_type == b"stsd" and b"avc1" in payload: + codec_name = "h264" + + metadata = { + "codec_name": codec_name, + "width": width, + "height": height, + "frame_count": frame_count, + } + expected = { + "codec_name": "h264", + "width": 16, + "height": 16, + "frame_count": 1, + } + if video_handler_count != 1 or metadata != expected: + raise BenchmarkContractError( + f"video probe MP4 must contain one 16x16 one-frame H.264 stream: {metadata}" + ) + return metadata + + +def _tiny_mp4_bytes() -> bytes: + """Return the validated deterministic one-frame MP4 probe fixture.""" + fixture = base64.b64decode("".join(_TINY_MP4_BASE64.split()), validate=True) + if hashlib.sha256(fixture).hexdigest() != VIDEO_PROBE_FIXTURE_SHA256: + raise BenchmarkContractError("video probe MP4 checksum does not match") + validate_video_probe_fixture(fixture) + return fixture + + +def _tiny_wav_bytes() -> bytes: + """Return a deterministic 10ms silent mono WAV used by the audio probes.""" + buffer = io.BytesIO() + with wave.open(buffer, "wb") as handle: + handle.setnchannels(1) + handle.setsampwidth(2) + handle.setframerate(8000) + handle.writeframes(b"\x00\x00" * 80) + return buffer.getvalue() + + +def _chat_probe_body(model_id: str, content: Any) -> bytes: + """Serialize a minimal single-message chat probe for ``model_id``.""" + payload = { + "model": model_id, + "messages": [{"role": "user", "content": content}], + "max_tokens": 1, + "temperature": 0.0, + } + return json.dumps(payload).encode("utf-8") + + +def _multipart_transcription_body(model_id: str) -> bytes: + """Build a deterministic multipart body for the audio transcription probe.""" + boundary = _MULTIPART_BOUNDARY.encode("ascii") + parts = [ + b"--" + boundary, + b'Content-Disposition: form-data; name="model"', + b"", + model_id.encode("utf-8"), + b"--" + boundary, + b'Content-Disposition: form-data; name="file"; filename="probe.wav"', + b"Content-Type: audio/wav", + b"", + _tiny_wav_bytes(), + b"--" + boundary + b"--", + b"", + ] + return b"\r\n".join(parts) + + +def _has_choice(payload: dict[str, Any]) -> bool: + """True when an OpenAI chat/completions payload carries at least one choice.""" + choices = payload.get("choices") + if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict): + return False + first = choices[0] + return isinstance(first.get("text"), str) or ( + isinstance(first.get("message"), dict) + and isinstance(first["message"].get("content"), str) + ) + + +def _has_embedding(payload: dict[str, Any]) -> bool: + """True when an embeddings payload carries at least one embedding vector.""" + data = payload.get("data") + return ( + isinstance(data, list) + and len(data) > 0 + and isinstance(data[0], dict) + and isinstance(data[0].get("embedding"), list) + ) + + +def _has_response_output(payload: dict[str, Any]) -> bool: + """True when a Responses API payload carries an output field.""" + return any(key in payload for key in ("output", "output_text", "response")) + + +def _has_transcription_text(payload: dict[str, Any]) -> bool: + """True when a transcription payload carries the transcribed text field.""" + return isinstance(payload.get("text"), str) + + +def _has_audio_signature(body: bytes) -> bool: + """Return whether a bounded response starts with a common audio container.""" + return ( + body.startswith((b"ID3", b"OggS", b"fLaC")) + or (body.startswith(b"RIFF") and body[8:12] == b"WAVE") + or (len(body) >= 2 and body[0] == 0xFF and body[1] & 0xE0 == 0xE0) + ) + + +def _image_data_uri() -> str: + """Data URI of the tiny PNG used by the image-understanding probe.""" + return f"data:image/png;base64,{_TINY_PNG_BASE64}" + + +def _video_data_uri() -> str: + """Return a data URI containing the validated one-frame MP4 fixture.""" + return ( + f"data:video/mp4;base64,{base64.b64encode(_tiny_mp4_bytes()).decode('ascii')}" + ) + + +def _audio_probe_base64() -> str: + """Base64 WAV payload used by the omni-style audio-understanding probe.""" + return base64.b64encode(_tiny_wav_bytes()).decode("ascii") + + +def _build_capability_probes() -> dict[str, dict[str, Any]]: + """Registry of every probe contract, in the fixed order they are attempted. + + Each spec: ``path``, ``content_type``, ``body`` (model_id -> bytes), + ``validate`` (decoded JSON -> bool), and ``binary_response`` for endpoints + that answer with raw media instead of JSON. A deterministic validated media fixture is used for each modality; only an + HTTP 200 with the expected response shape counts as contract support. + """ + return { + "chat_completion": { + "path": "/chat/completions", + "content_type": "application/json", + "body": lambda model_id: _chat_probe_body(model_id, "Reply with OK."), + "validate": _has_choice, + "binary_response": False, + }, + "text_completion": { + "path": "/completions", + "content_type": "application/json", + "body": lambda model_id: json.dumps( + {"model": model_id, "prompt": "OK", "max_tokens": 1, "temperature": 0.0} + ).encode("utf-8"), + "validate": _has_choice, + "binary_response": False, + }, + "response_generation": { + "path": "/responses", + "content_type": "application/json", + "body": lambda model_id: json.dumps( + {"model": model_id, "input": "Reply with OK.", "max_output_tokens": 16} + ).encode("utf-8"), + "validate": _has_response_output, + "binary_response": False, + }, + "text_embedding": { + "path": "/embeddings", + "content_type": "application/json", + "body": lambda model_id: json.dumps( + {"model": model_id, "input": "probe"} + ).encode("utf-8"), + "validate": _has_embedding, + "binary_response": False, + }, + "image_understanding": { + "path": "/chat/completions", + "content_type": "application/json", + "body": lambda model_id: _chat_probe_body( + model_id, + [ + {"type": "text", "text": "Describe the image in one word."}, + {"type": "image_url", "image_url": {"url": _image_data_uri()}}, + ], + ), + "validate": _has_choice, + "binary_response": False, + }, + "video_understanding": { + "path": "/chat/completions", + "content_type": "application/json", + "body": lambda model_id: _chat_probe_body( + model_id, + [ + {"type": "text", "text": "Describe the video in one word."}, + {"type": "video_url", "video_url": {"url": _video_data_uri()}}, + ], + ), + "validate": _has_choice, + "binary_response": False, + }, + "audio_understanding": { + "path": "/chat/completions", + "content_type": "application/json", + "body": lambda model_id: _chat_probe_body( + model_id, + [ + {"type": "text", "text": "Transcribe the audio."}, + { + "type": "input_audio", + "input_audio": {"data": _audio_probe_base64(), "format": "wav"}, + }, + ], + ), + "validate": _has_choice, + "binary_response": False, + }, + "audio_transcription": { + "path": "/audio/transcriptions", + "content_type": f"multipart/form-data; boundary={_MULTIPART_BOUNDARY}", + "body": _multipart_transcription_body, + "validate": _has_transcription_text, + "binary_response": False, + }, + "audio_speech": { + "path": "/audio/speech", + "content_type": "application/json", + "body": lambda model_id: json.dumps( + {"model": model_id, "input": "OK", "voice": "default"} + ).encode("utf-8"), + "validate": lambda payload: True, + "binary_response": True, + }, + } + + +CAPABILITY_PROBES = _build_capability_probes() +CAPABILITY_PROBE_ORDER = tuple(CAPABILITY_PROBES) + +# HTTP statuses meaning "this model does not serve this contract" (not an outage). +_UNSUPPORTED_HTTP_STATUS = frozenset({400, 404, 405, 415, 422, 501}) + + +def classify_probe_status(status: int) -> str: + """Map one probe HTTP status to its machine-readable outcome class.""" + if status == 200: + return "supported" + if status in _UNSUPPORTED_HTTP_STATUS: + return "unsupported" + if status in (401, 403): + return "auth_rejected" + if status == 408: + return "timeout" + if status == 429: + return "rate_limited" + if status >= 500: + return "unavailable" + return "failed" + + +def execute_capability_probe( + transport: ProviderTransport, + endpoint: str, + api_key: str, + model_id: str, + capability_name: str, + timer: Callable[[], float] = time.perf_counter, +) -> dict[str, Any]: + """Run one capability probe against one model and classify the outcome.""" + spec = CAPABILITY_PROBES[capability_name] + url = f"{endpoint.rstrip('/')}{spec['path']}" + headers = _auth_headers(api_key, spec["content_type"]) + started = timer() + try: + status, body = transport("POST", url, headers, spec["body"](model_id)) + except (TimeoutError, socket.timeout) as exc: + return _probe_row( + capability_name, + "timeout", + f"network_timeout:{type(exc).__name__}", + None, + started, + timer, + ) + except (urllib.error.URLError, ConnectionError) as exc: + return _probe_row( + capability_name, + "failed", + f"network_error:{type(exc).__name__}", + None, + started, + timer, + ) + + outcome = classify_probe_status(status) + if outcome == "auth_rejected": + raise BenchmarkAuthError( + f"provider rejected the benchmark credential during probes (HTTP {status})" + ) + reason = f"http_status:{status}" + if outcome == "supported" and not spec["binary_response"]: + try: + payload = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + payload = None + if not isinstance(payload, dict) or not spec["validate"](payload): + outcome, reason = ( + "malformed_response", + "http_200_with_unexpected_body_shape", + ) + if ( + outcome == "supported" + and spec["binary_response"] + and not _has_audio_signature(body) + ): + outcome, reason = "malformed_response", "http_200_without_audio_signature" + return _probe_row(capability_name, outcome, reason, status, started, timer) + + +def _probe_row( + capability_name: str, + probe_outcome: str, + outcome_reason: str, + http_status: int | None, + started: float, + timer: Callable[[], float], +) -> dict[str, Any]: + """Assemble one probe result row with its end-to-end latency.""" + return { + "capability_name": capability_name, + "probe_outcome": probe_outcome, + "outcome_reason": outcome_reason, + "http_status": http_status, + "probe_latency_ms": round((timer() - started) * 1000, 2), + } + + +_CHAT_CLASSIFICATIONS = frozenset( + {"chat_capable", "vision_chat_capable", "omni_capable"} +) + + +def classify_model_capabilities(probe_rows: list[dict[str, Any]]) -> dict[str, Any]: + """Derive one model-level classification from its per-capability probe rows. + + Chat-family support wins (with vision/omni refinements derived from the + modality probes); otherwise the strongest single-contract class applies; + otherwise the dominant failure mode is reported, so a skipped or throttled + model is never silently confused with an unsupported one. + """ + outcomes = {row["capability_name"]: row["probe_outcome"] for row in probe_rows} + supported = sorted( + name for name, outcome in outcomes.items() if outcome == "supported" + ) + supported_set = set(supported) + if "chat_completion" in supported_set: + if {"image_understanding", "audio_understanding"} <= supported_set: + classification = "omni_capable" + elif supported_set & {"image_understanding", "video_understanding"}: + classification = "vision_chat_capable" + else: + classification = "chat_capable" + elif "text_embedding" in supported_set: + classification = "embedding_only" + elif "text_completion" in supported_set: + classification = "completion_only" + elif "response_generation" in supported_set: + classification = "responses_only" + elif supported_set & {"audio_transcription", "audio_speech"}: + classification = "audio_only" + else: + observed = set(outcomes.values()) + if observed == {"skipped"}: + classification = "skipped" + elif "rate_limited" in observed: + classification = "rate_limited" + elif "unavailable" in observed: + classification = "unavailable" + elif observed & {"timeout", "failed", "malformed_response"}: + classification = "failed" + else: + classification = "unsupported_for_contract" + return { + "model_classification": classification, + "supported_capabilities": supported, + "chat_eligible": classification in _CHAT_CLASSIFICATIONS, + } + + +def probe_discovered_models( + models: list[dict[str, Any]], + transport: ProviderTransport, + endpoint: str, + api_key: str, + request_budget: RequestBudget, + probe_concurrency: int, + clock: Callable[[], float], + timer: Callable[[], float] = time.perf_counter, +) -> list[dict[str, Any]]: + """Probe every model with deterministic allocation and bounded concurrency. + + The permitted ``(model_id, capability)`` cells are fixed in sorted catalog + and capability order before worker threads start. Thread scheduling can + change completion order but cannot choose which cells run. + + Args: + models: Discovered model rows containing ``model_id`` and ``owned_by``. + transport: Provider request seam. + endpoint: OpenAI-compatible provider base endpoint. + api_key: In-memory credential value, never serialized. + request_budget: Shared hard provider-call cap. + probe_concurrency: Maximum simultaneous model workers. + clock: Provenance timestamp source. + timer: Per-probe monotonic latency source. + + Returns: + Sorted model rows with complete capability evidence for every model. + + Raises: + BenchmarkContractError: If concurrency is boolean or not positive. + """ + if ( + isinstance(probe_concurrency, bool) + or not isinstance(probe_concurrency, int) + or probe_concurrency < 1 + ): + raise BenchmarkContractError("probe_concurrency must be a positive integer") + + sorted_models = sorted(models, key=lambda row: row["model_id"]) + required_probe_requests = len(sorted_models) * len(CAPABILITY_PROBE_ORDER) + if required_probe_requests > request_budget.remaining_requests: + raise BenchmarkBudgetError( + f"complete capability probe plan needs {required_probe_requests} " + f"requests but only {request_budget.remaining_requests} remain" + ) + + discovered_at_unix = round(clock(), 3) + auth_rejected = threading.Event() + + def probe_one(model: dict[str, Any]) -> dict[str, Any]: + """Execute every preflighted capability cell for one model.""" + rows: list[dict[str, Any]] = [] + for capability_name in CAPABILITY_PROBE_ORDER: + if auth_rejected.is_set(): + raise BenchmarkAuthError( + "provider rejected the benchmark credential during probes" + ) + request_budget.spend_or_fail() + try: + rows.append( + execute_capability_probe( + transport, + endpoint, + api_key, + model["model_id"], + capability_name, + timer, + ) + ) + except BenchmarkAuthError: + auth_rejected.set() + raise + classified = classify_model_capabilities(rows) + return { + "model_id": model["model_id"], + "owned_by": model["owned_by"], + "endpoint": endpoint, + "discovered_at_unix": discovered_at_unix, + "capability_probe_rows": rows, + **classified, + } + + with ThreadPoolExecutor(max_workers=probe_concurrency) as executor: + results = list(executor.map(probe_one, sorted_models)) + return results + + +# -------------------------------------------------------------------------- +# Task manifest, scorers, pricing scenario +# -------------------------------------------------------------------------- + + +def score_exact_number_match(expected: dict[str, Any], answer_text: str) -> float: + """1.0 when the exact expected number appears as a standalone number in the answer. + + A trailing sentence period ("the answer is 21.") still matches; being part + of a longer number ("210", "21.5", "121") never does. + """ + target = decimal.Decimal(str(expected["number"])) + candidates = re.findall( + r"(? str: + """Canonicalize one expected text value for exact or containment checks.""" + normalized = value.strip() + return normalized if case_sensitive else normalized.casefold() + + +def _text_match_candidates(expected: dict[str, Any]) -> tuple[bool, tuple[str, ...]]: + """Return case sensitivity plus the declared expected text candidates.""" + case_sensitive = bool(expected.get("strict_case_sensitive")) + strict_texts = expected.get("strict_texts") + if isinstance(strict_texts, list) and strict_texts: + values = tuple(str(value) for value in strict_texts) + else: + values = (str(expected["substring"]),) + return case_sensitive, values + + +def expected_text_leaks(expected: dict[str, Any], prompt_text: str) -> bool: + """Return ``True`` when any declared expected text appears inside the prompt.""" + case_sensitive, candidates = _text_match_candidates(expected) + haystack = _normalize_expected_text(prompt_text, case_sensitive=case_sensitive) + for candidate in candidates: + if ( + _normalize_expected_text(candidate, case_sensitive=case_sensitive) + in haystack + ): + return True + return False + + +def score_substring_match(expected: dict[str, Any], answer_text: str) -> float: + """Score declared text answers with optional case-sensitive exact matching.""" + case_sensitive, candidates = _text_match_candidates(expected) + normalized_answer = _normalize_expected_text( + answer_text, case_sensitive=case_sensitive + ) + strict_texts = expected.get("strict_texts") + if isinstance(strict_texts, list) and strict_texts: + normalized_candidates = { + _normalize_expected_text(candidate, case_sensitive=case_sensitive) + for candidate in candidates + } + return 1.0 if normalized_answer in normalized_candidates else 0.0 + needle = _normalize_expected_text( + str(expected["substring"]), case_sensitive=case_sensitive + ) + return 1.0 if needle in normalized_answer else 0.0 + + +SCORER_REGISTRY: dict[tuple[str, str], Callable[[dict[str, Any], str], float]] = { + ("exact_number_match", "1"): score_exact_number_match, + ("substring_match", "1"): score_substring_match, +} + +_VALID_TASK_SPLITS = frozenset({"locked", "exploratory"}) + + +def _validate_expected( + scorer_key: tuple[str, str], expected: dict[str, Any], task_id: str +) -> None: + """Validate the scorer-specific expected-answer schema.""" + if scorer_key == ("exact_number_match", "1"): + number = expected.get("number") + if isinstance(number, bool) or not isinstance(number, (str, int, float)): + raise BenchmarkContractError( + f"task {task_id!r} exact-number scorer requires a finite 'number'" + ) + try: + if not decimal.Decimal(str(number)).is_finite(): + raise decimal.InvalidOperation + except decimal.InvalidOperation as exc: + raise BenchmarkContractError( + f"task {task_id!r} exact-number scorer requires a finite 'number'" + ) from exc + return + strict_texts = expected.get("strict_texts") + substring = expected.get("substring") + if not ( + isinstance(substring, str) + or ( + isinstance(strict_texts, list) + and bool(strict_texts) + and all(isinstance(value, str) for value in strict_texts) + ) + ): + raise BenchmarkContractError( + f"task {task_id!r} substring scorer requires 'substring' or non-empty string 'strict_texts'" + ) + + +def load_task_manifest(path: str) -> dict[str, Any]: + """Load and validate the versioned task manifest, rejecting leakage and drift. + + Enforces: a manifest version, unique immutable snake_case task ids, known + splits, registered scorer name+version pairs, and the no-leakage rule that + an expected answer value never appears inside its own task prompt. + """ + with open(path, "r", encoding="utf-8") as handle: + try: + manifest = json.load(handle) + except ValueError as exc: + raise BenchmarkContractError( + f"task manifest is not valid JSON: {exc}" + ) from exc + if not isinstance(manifest, dict) or not isinstance( + manifest.get("manifest_version"), str + ): + raise BenchmarkContractError( + "task manifest must be an object with a string 'manifest_version'" + ) + tasks = manifest.get("tasks") + if not isinstance(tasks, list) or not tasks: + raise BenchmarkContractError( + "task manifest must carry a non-empty 'tasks' list" + ) + seen_task_ids: set[str] = set() + for task in tasks: + if not isinstance(task, dict): + raise BenchmarkContractError("every task manifest entry must be an object") + task_id = task.get("task_id") + if not isinstance(task_id, str) or not is_two_word_snake_case(task_id): + raise BenchmarkContractError( + f"task_id must be two-plus-word snake_case: {task_id!r}" + ) + if task_id in seen_task_ids: + raise BenchmarkContractError(f"duplicate task_id in manifest: {task_id!r}") + seen_task_ids.add(task_id) + if task.get("split") not in _VALID_TASK_SPLITS: + raise BenchmarkContractError( + f"task {task_id!r} split must be 'locked' or 'exploratory'" + ) + prompt = task.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + raise BenchmarkContractError( + f"task {task_id!r} must carry a non-empty prompt" + ) + scorer = task.get("scorer") + if not isinstance(scorer, dict): + raise BenchmarkContractError(f"task {task_id!r} must carry a scorer object") + scorer_key = (str(scorer.get("name")), str(scorer.get("version"))) + if scorer_key not in SCORER_REGISTRY: + raise BenchmarkContractError( + f"task {task_id!r} names an unregistered scorer: {scorer_key}" + ) + expected = task.get("expected") + if not isinstance(expected, dict) or not expected: + raise BenchmarkContractError( + f"task {task_id!r} must carry a non-empty expected object" + ) + _validate_expected(scorer_key, expected, task_id) + # No-leakage rule, defined by the scorer itself: if the registered + # scorer would award the prompt text a point, the expected answer has + # leaked into the prompt and a prompt-echoing model would score. + if scorer_key == ("substring_match", "1"): + leaked = expected_text_leaks(expected, prompt) + else: + leaked = SCORER_REGISTRY[scorer_key](expected, prompt) != 0.0 + if leaked: + raise BenchmarkContractError( + f"task {task_id!r} leaks its expected answer into the prompt (test-set leakage)" + ) + return manifest + + +def locked_evaluation_tasks(manifest: dict[str, Any]) -> list[dict[str, Any]]: + """Return only the locked evaluation split, in manifest order.""" + return [task for task in manifest["tasks"] if task["split"] == "locked"] + + +def _require_finite_rate(value: Any, label: str) -> float: + """Validate one USD-per-million-token rate: a finite, non-negative number.""" + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + ): + raise BenchmarkContractError( + f"pricing scenario rate {label} must be a finite non-negative number" + ) + return float(value) + + +_REVIEWED_PRICING_FIELDS = ( + "source_url", + "reviewed_by", + "reviewed_at_date", + "valid_until_date", + "rate_basis", + "uncertainty", +) + + +def _parse_evidence_date(value: Any, field_name: str) -> datetime_module.date: + """Parse one ISO evidence date or raise a field-specific contract error.""" + if not isinstance(value, str): + raise BenchmarkContractError( + f"pricing scenario {field_name} must be an ISO date string" + ) + try: + return datetime_module.date.fromisoformat(value) + except ValueError as exc: + raise BenchmarkContractError( + f"pricing scenario {field_name} must be a valid ISO date" + ) from exc + + +def _validate_reviewed_pricing_metadata(scenario: dict[str, Any]) -> None: + """Require complete provenance for a scenario labeled ``reviewed``.""" + missing = [field for field in _REVIEWED_PRICING_FIELDS if field not in scenario] + if missing: + raise BenchmarkContractError( + f"reviewed pricing scenario is missing fields: {missing}" + ) + parsed_source = urllib.parse.urlparse(str(scenario["source_url"])) + if parsed_source.scheme != "https" or not parsed_source.hostname: + raise BenchmarkContractError( + "reviewed pricing scenario source_url must use https" + ) + for field_name in ("reviewed_by", "rate_basis", "uncertainty"): + value = scenario[field_name] + if not isinstance(value, str) or not value.strip(): + raise BenchmarkContractError( + f"reviewed pricing scenario {field_name} must be non-empty" + ) + reviewed_at = _parse_evidence_date(scenario["reviewed_at_date"], "reviewed_at_date") + valid_until = _parse_evidence_date(scenario["valid_until_date"], "valid_until_date") + if valid_until < reviewed_at: + raise BenchmarkContractError( + "reviewed pricing scenario valid_until_date precedes reviewed_at_date" + ) + + +def validate_live_pricing_scenario( + scenario: dict[str, Any] | None, + today: datetime_module.date | None = None, +) -> None: + """Fail before egress when supplied live price evidence is not current. + + Omitting a scenario is valid and leaves every hypothetical cost ``unknown``. + Supplying one requires an explicit reviewed status, complete provenance, and + a validity horizon that includes the run date. + """ + if scenario is None: + return + if scenario.get("scenario_status") != "reviewed": + raise BenchmarkContractError( + "live benchmark pricing scenario must be independently reviewed" + ) + _validate_reviewed_pricing_metadata(scenario) + observed_date = today or datetime_module.date.today() + reviewed_at = _parse_evidence_date(scenario["reviewed_at_date"], "reviewed_at_date") + valid_until = _parse_evidence_date(scenario["valid_until_date"], "valid_until_date") + if reviewed_at > observed_date: + raise BenchmarkContractError("reviewed pricing evidence is dated in the future") + if observed_date > valid_until: + raise BenchmarkContractError("reviewed pricing evidence expired") + + +def load_pricing_scenario(path: str | None) -> dict[str, Any] | None: + """Load and validate one explicit hypothetical price-assumption file. + + ``None`` is legal and keeps hypothetical costs ``unknown``. Rates are never + inferred: only finite non-negative input/output USD-per-million-token values + explicitly present in the supplied file are accepted. A scenario labeled + ``reviewed`` must also carry complete source and validity metadata. + + Args: + path: JSON scenario path, or ``None`` to omit paid-cost assumptions. + + Returns: + Validated scenario dictionary or ``None``. + + Raises: + BenchmarkContractError: If JSON, status, provenance, or rates are invalid. + """ + if path is None: + return None + with open(path, "r", encoding="utf-8") as handle: + try: + scenario = json.load(handle) + except ValueError as exc: + raise BenchmarkContractError( + f"pricing scenario is not valid JSON: {exc}" + ) from exc + if not isinstance(scenario, dict) or not isinstance( + scenario.get("scenario_version"), str + ): + raise BenchmarkContractError( + "pricing scenario must be an object with a string 'scenario_version'" + ) + if scenario.get("scenario_status") not in ("example_unreviewed", "reviewed"): + raise BenchmarkContractError( + "pricing scenario_status must be 'example_unreviewed' or 'reviewed'" + ) + rates = scenario.get("usd_per_million_tokens") + if not isinstance(rates, dict): + raise BenchmarkContractError( + "pricing scenario must carry a 'usd_per_million_tokens' object" + ) + for model_id, rate in rates.items(): + if not isinstance(model_id, str) or not model_id.strip(): + raise BenchmarkContractError("pricing model id must be a non-empty string") + if not isinstance(rate, dict): + raise BenchmarkContractError( + f"pricing entry for {model_id!r} must be an object" + ) + _require_finite_rate(rate.get("input"), f"{model_id}.input") + _require_finite_rate(rate.get("output"), f"{model_id}.output") + if scenario["scenario_status"] == "reviewed": + _validate_reviewed_pricing_metadata(scenario) + return scenario + + +def hypothetical_cost_usd( + pricing_scenario: dict[str, Any] | None, + usage_by_model: dict[str, dict[str, int]], +) -> float | str: + """Cost under the pricing scenario, or ``"unknown"`` when any model is unpriced. + + ``usage_by_model`` maps model id to its prompt/completion token counts for + one cell. No authoritative rate for a used model means the whole cell is + honestly ``"unknown"`` — a partial sum would understate cost. + """ + if pricing_scenario is None: + return "unknown" + rates = pricing_scenario["usd_per_million_tokens"] + total = 0.0 + for model_id, usage in usage_by_model.items(): + rate = rates.get(model_id) + if rate is None: + return "unknown" + total += usage["prompt_tokens"] * float(rate["input"]) / 1_000_000 + total += usage["completion_tokens"] * float(rate["output"]) / 1_000_000 + return round(total, 10) + + +# -------------------------------------------------------------------------- +# Policy evaluation +# -------------------------------------------------------------------------- + + +def sanitize_worker_agent_id(model_id: str, taken_ids: set[str]) -> str: + """Deterministically derive a convention-compliant agent id from a model id.""" + base = re.sub(r"[^a-z0-9]+", "_", model_id.lower()).strip("_") or "unnamed_model" + if not is_two_word_snake_case(base): + base = f"nim_{base}" + candidate = base + suffix = 2 + while candidate in taken_ids: + candidate = f"{base}_{suffix}" + suffix += 1 + taken_ids.add(candidate) + return candidate + + +def build_worker_agents( + probed_models: list[dict[str, Any]], + base_url: str, + max_eval_models: int, +) -> list[ModelAgent]: + """Build the evaluation worker pool from chat-eligible probed models. + + Deterministic: models are already sorted by id; the pool is capped at + ``max_eval_models`` so a huge catalog cannot silently explode the budget. + """ + if max_eval_models < 1: + raise BenchmarkContractError("max_eval_models must be a positive integer") + taken_ids: set[str] = set() + agents: list[ModelAgent] = [] + for row in probed_models: + if not row["chat_eligible"]: + continue + if len(agents) >= max_eval_models: + break + agents.append( + ModelAgent( + id=sanitize_worker_agent_id(row["model_id"], taken_ids), + model=row["model_id"], + base_url=base_url, + credential_key=NIM_CREDENTIAL_NAME, + tags=("reasoning", "writing"), + ) + ) + return agents + + +def _coerce_token_count(value: Any) -> int | None: + """Defensively coerce a provider-reported token count; ``None`` when unusable. + + Guards the adversarial cases: booleans, non-numbers, NaN/inf, negatives. + """ + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + if not math.isfinite(value) or value < 0: + return None + return int(value) + + +def _cell_usage( + trace: list[dict[str, Any]], + agents_by_id: dict[str, str], + task_prompt: str, +) -> tuple[dict[str, dict[str, int]], dict[str, Any]]: + """Aggregate per-model token usage for one cell, labeling its source honestly. + + Provider-reported usage wins; steps without usable reported numbers fall + back to the repo's character-length estimate and mark the whole cell + ``estimated`` (never silently mixed into ``reported``). + """ + usage_by_model: dict[str, dict[str, int]] = {} + any_estimated = False + models_used: list[dict[str, Any]] = [] + for row in trace: + agent_id = row.get("served_agent_id") or row["agent_id"] + try: + model_id = agents_by_id[agent_id] + except (KeyError, TypeError) as exc: + raise BenchmarkContractError( + f"trace references unknown agent {agent_id!r}" + ) from exc + models_used.append( + { + "step_id": row["id"], + "role": row["role"], + "agent_id": agent_id, + "model_id": model_id, + } + ) + usage = row.get("usage") if isinstance(row.get("usage"), dict) else {} + prompt_tokens = _coerce_token_count(usage.get("prompt_tokens")) + completion_tokens = _coerce_token_count(usage.get("completion_tokens")) + if prompt_tokens is None: + prompt_tokens = estimate_tokens(task_prompt) + any_estimated = True + if completion_tokens is None: + completion_tokens = estimate_tokens(row.get("output") or "") + any_estimated = True + bucket = usage_by_model.setdefault( + model_id, {"prompt_tokens": 0, "completion_tokens": 0} + ) + bucket["prompt_tokens"] += prompt_tokens + bucket["completion_tokens"] += completion_tokens + prompt_total = sum(bucket["prompt_tokens"] for bucket in usage_by_model.values()) + completion_total = sum( + bucket["completion_tokens"] for bucket in usage_by_model.values() + ) + summary = { + "prompt_tokens": prompt_total, + "completion_tokens": completion_total, + "total_tokens": prompt_total + completion_total, + "token_usage_source": "estimated" if any_estimated else "reported", + "models_used": models_used, + } + return usage_by_model, summary + + +def _classify_run_error(exc: Exception) -> str: + """Split a failed policy run into the issue's timeout vs failure classes.""" + causes = {type(exc), type(exc.__cause__)} + if causes & {TimeoutError, socket.timeout}: + return "timeout" + return "failure" + + +def _run_error_reason(exc: Exception) -> str: + """Return a bounded, redacted category and detail for a failed policy cell.""" + if isinstance(exc, urllib.error.HTTPError): + return f"provider_http_error:{exc.code}" + if isinstance(exc, PolicyTokenBudgetExceeded): + return "policy_token_budget_exceeded" + detail = redact_text(str(exc)).strip().replace("\n", " ")[:256] + category = type(exc).__name__ + return f"{category}:{detail}" if detail else category + + +def run_policy_cell( + policy_name: str, + task: dict[str, Any], + run_callable: Callable[[], dict[str, Any]], + agents_by_id: dict[str, str], + pricing_scenario: dict[str, Any] | None, + timer: Callable[[], float], + failure_evidence: Callable[[], dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Execute one policy on one task and record the full evidence cell.""" + scorer = task["scorer"] + started = timer() + try: + result = run_callable() + except (BenchmarkContractError, BenchmarkBudgetError, BenchmarkAuthError): + # Budget exhaustion and credential rejection must abort the whole run + # (fail closed), never degrade into one quietly failed cell. + raise + except Exception as exc: # noqa: BLE001 - classified into the contract outcomes + incurred = failure_evidence() if failure_evidence is not None else {} + prompt_tokens = incurred.get("prompt_tokens", 0) + completion_tokens = incurred.get("completion_tokens", 0) + total_tokens = incurred.get( + "total_tokens", prompt_tokens + completion_tokens + ) + return { + "policy_name": policy_name, + "task_id": task["task_id"], + "task_split": task["split"], + "scorer_name": scorer["name"], + "scorer_version": scorer["version"], + "task_score": None, + "run_outcome": _classify_run_error(exc), + "outcome_reason": _run_error_reason(exc), + "end_to_end_latency_ms": round((timer() - started) * 1000, 3), + "provider_latency_ms": None, + "call_count": incurred.get("call_count", 0), + "workflow_depth": incurred.get("call_count", 0), + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens, + "token_usage_source": "estimated" if incurred else "unavailable", + "actual_cost_usd": 0.0, + "hypothetical_cost_usd": "unknown", + "models_used": incurred.get("models_used", []), + "response_sha256": None, + } + elapsed_ms = round((timer() - started) * 1000, 3) + answer = result.get("answer") or "" + scorer_fn = SCORER_REGISTRY[(scorer["name"], scorer["version"])] + trace = result.get("trace") or [] + usage_by_model, usage_summary = _cell_usage(trace, agents_by_id, task["prompt"]) + return { + "policy_name": policy_name, + "task_id": task["task_id"], + "task_split": task["split"], + "scorer_name": scorer["name"], + "scorer_version": scorer["version"], + "task_score": scorer_fn(task["expected"], answer), + "run_outcome": "success", + "outcome_reason": "completed", + "end_to_end_latency_ms": elapsed_ms, + # Provider-side latency is not observable through the OpenAI-compatible + # response body; recorded as None rather than a fabricated number. + "provider_latency_ms": None, + "call_count": len(trace), + "workflow_depth": len(trace), + "prompt_tokens": usage_summary["prompt_tokens"], + "completion_tokens": usage_summary["completion_tokens"], + "total_tokens": usage_summary["total_tokens"], + "token_usage_source": usage_summary["token_usage_source"], + # Actual cost of the hosted NIM catalog to the caller is zero today; + # hypothetical paid cost comes only from the explicit scenario. + "actual_cost_usd": 0.0, + "hypothetical_cost_usd": hypothetical_cost_usd( + pricing_scenario, usage_by_model + ), + "models_used": usage_summary["models_used"], + "response_sha256": hashlib.sha256(answer.encode("utf-8")).hexdigest(), + } + + +def _combined_rate(pricing_scenario: dict[str, Any], model_id: str) -> float | None: + """Combined input+output USD/1M rate for cheapest-worker selection, or ``None``.""" + rate = pricing_scenario["usd_per_million_tokens"].get(model_id) + if rate is None: + return None + return float(rate["input"]) + float(rate["output"]) + + +def cheapest_priced_agent( + agents: list[ModelAgent], pricing_scenario: dict[str, Any] | None +) -> ModelAgent | None: + """The cheapest scenario-priced worker (deterministic tiebreak by model id).""" + if pricing_scenario is None: + return None + priced = [ + (rate, agent.model, agent) + for agent in agents + for rate in [_combined_rate(pricing_scenario, agent.model)] + if rate is not None + ] + if not priced: + return None + return min(priced, key=lambda row: (row[0], row[1]))[2] + + +def planned_evaluation_requests(worker_count: int, locked_task_count: int) -> int: + """Upper bound on evaluation calls, checked pre-flight so the run fails closed. + + Direct baselines, ``route_once``, and cheapest-eligible cells each reserve + one worker call plus one real-time judge call. ``route_once`` reserves the + full equal-call envelope because endpoint races and future failover may use + more than one worker attempt. ``conduct`` reserves its five-step workflow + envelope, including the model judge. + """ + return locked_task_count * ( + worker_count * 2 + MAX_WORKFLOW_DEPTH + MAX_WORKFLOW_DEPTH + 2 + ) + + +def plan_complete_request_budget( + discovered_model_count: int, + max_eval_models: int, + locked_task_count: int, +) -> dict[str, int]: + """Return the complete conservative request plan for one catalog snapshot. + + The plan reserves one catalog request, every model-capability probe, and + the worst-case equal-budget evaluation envelope for every worker that may + enter the capped evaluation pool. It is intentionally conservative: fewer + chat-eligible or scenario-priced workers may leave requests unused, but a + live run never starts a biased partial probe phase. + + Args: + discovered_model_count: Usable model ids returned by ``/v1/models``. + max_eval_models: Maximum workers allowed into policy evaluation. + locked_task_count: Number of locked benchmark tasks. + + Returns: + Named request counts including the complete run total. + + Raises: + BenchmarkContractError: If any count is boolean or not positive. + """ + counts = { + "discovered_model_count": discovered_model_count, + "max_eval_models": max_eval_models, + "locked_task_count": locked_task_count, + } + for label, value in counts.items(): + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise BenchmarkContractError(f"{label} must be a positive integer") + planned_worker_count = min(discovered_model_count, max_eval_models) + capability_probe_request_count = discovered_model_count * len( + CAPABILITY_PROBE_ORDER + ) + evaluation_reserve_request_count = planned_evaluation_requests( + planned_worker_count, + locked_task_count, + ) + return { + "catalog_request_count": 1, + "capability_probe_request_count": capability_probe_request_count, + "evaluation_reserve_request_count": evaluation_reserve_request_count, + "planned_worker_count": planned_worker_count, + "total_required_request_count": ( + 1 + capability_probe_request_count + evaluation_reserve_request_count + ), + } + + +def planned_complete_run_requests( + model_count: int, + locked_task_count: int, + max_eval_models: int, +) -> dict[str, int]: + """Return buyer-facing request counts for a complete benchmark run. + + This stable planning view translates the internal conservative preflight + into terminology used by release acceptance, operator documentation, and + acquisition evidence. Validation remains centralized in + :func:`plan_complete_request_budget`, so both views fail closed identically. + + Args: + model_count: Usable model identifiers discovered from ``/v1/models``. + locked_task_count: Number of locked evaluation tasks. + max_eval_models: Maximum workers admitted to policy comparison. + + Returns: + Catalog, capability, evaluation, post-catalog, and total request counts. + """ + plan = plan_complete_request_budget( + discovered_model_count=model_count, + max_eval_models=max_eval_models, + locked_task_count=locked_task_count, + ) + requests_after_catalog = ( + plan["capability_probe_request_count"] + + plan["evaluation_reserve_request_count"] + ) + return { + "catalog_discovery_requests": plan["catalog_request_count"], + "capability_probe_requests": plan["capability_probe_request_count"], + "evaluation_worker_ceiling": plan["planned_worker_count"], + "evaluation_requests": plan["evaluation_reserve_request_count"], + "requests_after_catalog": requests_after_catalog, + "total_requests": plan["total_required_request_count"], + } + + +def evaluate_policies( + agents: list[ModelAgent], + manifest: dict[str, Any], + pricing_scenario: dict[str, Any] | None, + client: ModelClient, + request_budget: RequestBudget, + timer: Callable[[], float] = time.perf_counter, + total_token_budget: int = DEFAULT_POLICY_TOTAL_TOKEN_BUDGET, + maximum_calls: int = MAX_WORKFLOW_DEPTH, +) -> dict[str, Any]: + """Run every compared policy with equal cell-level token and call budgets. + + Every policy/task cell receives a fresh orchestrator and limiter so traces, + usage, call counts, and allowances never bleed across tasks or policy arms. + + Args: + agents: Chat-eligible workers selected from capability probes. + manifest: Validated task manifest. + pricing_scenario: Optional explicit hypothetical price assumptions. + client: Shared request-budgeted model client. + request_budget: Complete-run provider request cap. + timer: Monotonic latency source. + total_token_budget: Equal prompt-plus-completion allowance per cell. + maximum_calls: Equal declared provider-call envelope per cell. + + Returns: + Evaluation cells and pool/task metadata. + + Raises: + BenchmarkContractError: If no workers or locked tasks are available. + BenchmarkBudgetError: If the complete evaluation cannot fit the run cap. + """ + if not agents: + raise BenchmarkContractError( + "policy evaluation requires at least one chat-eligible worker" + ) + tasks = locked_evaluation_tasks(manifest) + if not tasks: + raise BenchmarkContractError("task manifest has no locked evaluation tasks") + planned = planned_evaluation_requests(len(agents), len(tasks)) + if planned > request_budget.remaining_requests: + raise BenchmarkBudgetError( + f"planned evaluation needs up to {planned} requests but only " + f"{request_budget.remaining_requests} remain in the budget" + ) + + agents_by_id = {agent.id: agent.model for agent in agents} + depth_policy = OrchestrationPolicy( + route_p95_seconds=2.5, + realtime_judge=True, + verifier_required=True, + workflow_planning="template", + max_workflow_steps=MAX_WORKFLOW_DEPTH, + verifier_judge="model", + ) + + def run_cell( + policy_name: str, + task: dict[str, Any], + pool: list[ModelAgent], + mode: str, + ) -> dict[str, Any]: + """Run one independent policy/task cell and append budget evidence.""" + cell_client = EqualBudgetModelClient( + client, + total_token_budget, + maximum_calls, + ) + orchestrator = TaskOrchestrator( + pool, + client=cell_client, + tool_retry_attempts=0, + ) + orchestrator.policy = depth_policy + def complete_cell() -> dict[str, Any]: + try: + result = orchestrator.complete( + [{"role": "user", "content": task["prompt"]}], + mode=mode, + ) + except Exception: + if cell_client.contract_error is not None: + raise cell_client.contract_error + raise + if cell_client.contract_error is not None: + raise cell_client.contract_error + return result + + cell = run_policy_cell( + policy_name, + task, + complete_cell, + agents_by_id, + pricing_scenario, + timer, + lambda: { + "call_count": cell_client.observed_calls, + "prompt_tokens": cell_client.observed_prompt_tokens, + "completion_tokens": cell_client.observed_completion_tokens, + "total_tokens": cell_client.observed_tokens, + "models_used": cell_client.attempted_models, + }, + ) + cell.update( + { + "configured_total_token_budget": total_token_budget, + "configured_maximum_calls": maximum_calls, + "observed_budget_tokens": cell_client.observed_tokens, + "observed_budget_calls": cell_client.observed_calls, + "remaining_budget_tokens": cell_client.remaining_tokens, + } + ) + if cell["token_usage_source"] == "estimated" and cell_client.observed_calls: + cell.update( + { + "prompt_tokens": cell_client.observed_prompt_tokens, + "completion_tokens": cell_client.observed_completion_tokens, + "total_tokens": cell_client.observed_tokens, + "hypothetical_cost_usd": hypothetical_cost_usd( + pricing_scenario, cell_client.estimated_usage_by_model + ), + } + ) + if cell_client.exceeded: + cell["run_outcome"] = "failure" + cell["outcome_reason"] = "observed_usage_exceeded_equal_token_budget" + cell["task_score"] = None + return cell + + cells: list[dict[str, Any]] = [] + for agent in agents: + for task in tasks: + cells.append( + run_cell( + f"direct_single_worker:{agent.model}", + task, + [agent], + "route", + ) + ) + for task in tasks: + cells.append(run_cell("route_once", task, agents, "route")) + cells.append(run_cell("conduct_bounded", task, agents, "conduct")) + + cheapest_skip_reason = None + cheapest = cheapest_priced_agent(agents, pricing_scenario) + if cheapest is None: + cheapest_skip_reason = ( + "no_pricing_scenario_supplied" + if pricing_scenario is None + else "no_worker_priced_by_scenario" + ) + else: + for task in tasks: + cells.append( + run_cell( + "cheapest_eligible_worker", + task, + [cheapest], + "route", + ) + ) + cells.sort(key=lambda cell: (cell["policy_name"], cell["task_id"])) + return { + "evaluation_cells": cells, + "cheapest_worker_skip_reason": cheapest_skip_reason, + "locked_task_count": len(tasks), + "worker_count": len(agents), + } + + +# -------------------------------------------------------------------------- +# Statistics: paired bootstrap + Pareto frontiers +# -------------------------------------------------------------------------- + + +def paired_bootstrap_mean_difference( + paired_scores: list[tuple[float, float]], + iterations: int = 2000, + seed: int = 7, +) -> dict[str, Any]: + """Paired bootstrap CI for mean(score_a - score_b) over shared tasks.""" + if not paired_scores: + raise BenchmarkContractError( + "paired bootstrap requires at least one score pair" + ) + differences = [a - b for a, b in paired_scores] + rng = random.Random(seed) + resampled_means = sorted( + sum(rng.choice(differences) for _ in differences) / len(differences) + for _ in range(iterations) + ) + lower_index = int(0.025 * (iterations - 1)) + upper_index = int(0.975 * (iterations - 1)) + return { + "mean_difference": round(sum(differences) / len(differences), 6), + "ci_low": round(resampled_means[lower_index], 6), + "ci_high": round(resampled_means[upper_index], 6), + "iterations": iterations, + "seed": seed, + "pair_count": len(differences), + "method": "paired_bootstrap_percentile_95", + } + + +def pareto_frontier( + rows: list[dict[str, Any]], quality_key: str, cost_key: str +) -> list[dict[str, Any]]: + """Rows not dominated on (``quality_key`` up, ``cost_key`` down).""" + frontier = [ + a + for a in rows + if not any( + b is not a + and b[quality_key] >= a[quality_key] + and b[cost_key] <= a[cost_key] + and (b[quality_key] > a[quality_key] or b[cost_key] < a[cost_key]) + for b in rows + ) + ] + return sorted(frontier, key=lambda row: (-row[quality_key], row[cost_key])) + + +def summarize_policies(cells: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Aggregate evaluation cells per policy with honest unknown-cost labeling.""" + grouped: dict[str, list[dict[str, Any]]] = {} + for cell in cells: + grouped.setdefault(cell["policy_name"], []).append(cell) + summaries = [] + for policy_name in sorted(grouped): + policy_cells = grouped[policy_name] + scored = [cell for cell in policy_cells if cell["run_outcome"] == "success"] + priced = [ + cell + for cell in policy_cells + if isinstance(cell["hypothetical_cost_usd"], float) + ] + mean_score = round( + sum(cell["task_score"] for cell in scored) / len(policy_cells), 6 + ) + summaries.append( + { + "policy_name": policy_name, + "cell_count": len(policy_cells), + "success_count": len(scored), + "completion_fraction": round(len(scored) / len(policy_cells), 6), + "mean_task_score": mean_score, + "mean_latency_ms": round( + sum(cell["end_to_end_latency_ms"] for cell in policy_cells) + / len(policy_cells), + 3, + ), + "total_call_count": sum(cell["call_count"] for cell in policy_cells), + "max_workflow_depth": max( + cell["workflow_depth"] for cell in policy_cells + ), + "total_tokens": sum(cell["total_tokens"] for cell in policy_cells), + "actual_cost_usd": 0.0, + "mean_hypothetical_cost_usd": ( + round( + sum(cell["hypothetical_cost_usd"] for cell in priced) + / len(priced), + 10, + ) + if len(priced) == len(policy_cells) + else "unknown" + ), + "unknown_hypothetical_cost_cells": len(policy_cells) - len(priced), + } + ) + return summaries + + +def best_single_worker_hindsight( + summaries: list[dict[str, Any]], +) -> dict[str, Any] | None: + """The best direct single worker selected in hindsight on the locked split.""" + direct = [ + row + for row in summaries + if row["policy_name"].startswith("direct_single_worker:") + ] + if not direct: + return None + best = max(direct, key=lambda row: (row["mean_task_score"], row["policy_name"])) + return { + "policy_name": best["policy_name"], + "model_id": best["policy_name"].split(":", 1)[1], + "mean_task_score": best["mean_task_score"], + "selection_basis": "hindsight_argmax_mean_locked_score", + } + + +def paired_policy_comparisons( + cells: list[dict[str, Any]], seed: int +) -> list[dict[str, Any]]: + """Paired task-level bootstrap comparisons between the headline policies.""" + scores: dict[str, dict[str, float]] = {} + for cell in cells: + if cell["run_outcome"] == "success": + scores.setdefault(cell["policy_name"], {})[cell["task_id"]] = cell[ + "task_score" + ] + summaries = summarize_policies(cells) + hindsight = best_single_worker_hindsight(summaries) + comparison_pairs = [ + ("conduct_bounded", "route_once"), + ("cheapest_eligible_worker", "route_once"), + ] + if hindsight is not None: + comparison_pairs.append(("route_once", hindsight["policy_name"])) + comparison_pairs.append(("conduct_bounded", hindsight["policy_name"])) + comparisons = [] + for policy_a, policy_b in comparison_pairs: + tasks_a, tasks_b = scores.get(policy_a), scores.get(policy_b) + if not tasks_a or not tasks_b: + continue + shared_tasks = sorted(set(tasks_a) & set(tasks_b)) + if not shared_tasks: + continue + pairs = [(tasks_a[task_id], tasks_b[task_id]) for task_id in shared_tasks] + comparisons.append( + { + "policy_a": policy_a, + "policy_b": policy_b, + **paired_bootstrap_mean_difference(pairs, seed=seed), + } + ) + return comparisons + + +def _numeric_cost_rows(summaries: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Summaries whose mean hypothetical cost is numeric (unknowns excluded, labeled).""" + return [ + row for row in summaries if isinstance(row["mean_hypothetical_cost_usd"], float) + ] + + +def build_pareto_frontiers(summaries: list[dict[str, Any]]) -> dict[str, Any]: + """Quality-latency and quality-hypothetical-cost Pareto frontiers.""" + successful = [row for row in summaries if row["success_count"] > 0] + return { + "quality_vs_latency": pareto_frontier( + successful, + "mean_task_score", + "mean_latency_ms", + ), + "quality_vs_hypothetical_cost": pareto_frontier( + _numeric_cost_rows(successful), + "mean_task_score", + "mean_hypothetical_cost_usd", + ), + "excluded_unknown_cost_policies": sorted( + row["policy_name"] + for row in successful + if not isinstance(row["mean_hypothetical_cost_usd"], float) + ), + "excluded_zero_success_policies": sorted( + row["policy_name"] for row in summaries if row["success_count"] == 0 + ), + } + + +def _validate_actual_cost_evidence(report: dict[str, Any]) -> None: + """Require complete official provenance for the zero access-cost claim.""" + evidence = report.get("actual_cost_evidence") + if not isinstance(evidence, dict): + raise BenchmarkContractError("benchmark report is missing actual_cost_evidence") + required_fields = ( + "evidence_schema_version", + "source_title", + "source_url", + "reviewed_at_date", + "valid_until_date", + "access_program", + "access_scope", + "production_access_note", + "actual_cost_usd", + "uncertainty", + ) + missing = [field for field in required_fields if field not in evidence] + if missing: + raise BenchmarkContractError( + f"actual cost evidence is missing fields: {missing}" + ) + if evidence["actual_cost_usd"] != 0.0: + raise BenchmarkContractError( + "actual cost evidence must preserve the reviewed zero-cost value" + ) + if evidence["source_url"] != "https://docs.api.nvidia.com/nim/docs/product": + raise BenchmarkContractError( + "actual cost evidence must cite the reviewed NVIDIA NIM General FAQ" + ) + reviewed_at = _parse_evidence_date(evidence["reviewed_at_date"], "reviewed_at_date") + valid_until = _parse_evidence_date(evidence["valid_until_date"], "valid_until_date") + if valid_until < reviewed_at: + raise BenchmarkContractError( + "actual cost evidence validity precedes its review date" + ) + + +def _require_current_actual_cost_evidence( + today: datetime_module.date | None = None, +) -> None: + """Fail closed after the reviewed hosted-access validity horizon.""" + observed_date = today or datetime_module.date.today() + reviewed_at = _parse_evidence_date( + ACTUAL_COST_EVIDENCE["reviewed_at_date"], + "reviewed_at_date", + ) + valid_until = _parse_evidence_date( + ACTUAL_COST_EVIDENCE["valid_until_date"], + "valid_until_date", + ) + if observed_date < reviewed_at: + raise BenchmarkContractError( + "reviewed NVIDIA hosted-endpoint cost evidence is dated in the future" + ) + if observed_date > valid_until: + raise BenchmarkContractError( + "reviewed NVIDIA hosted-endpoint cost evidence expired; " + "re-review official terms" + ) + + +def _evaluation_evidence_summary( + cells: list[dict[str, Any]], + locked_task_count: int, +) -> dict[str, Any]: + """Classify whether benchmark evidence can inform production review.""" + headline_cells = [ + cell for cell in cells if cell["policy_name"] != "cheapest_eligible_worker" + ] + successful_cells = [ + cell for cell in headline_cells if cell["run_outcome"] == "success" + ] + completion_fraction = ( + round(len(successful_cells) / len(headline_cells), 6) if headline_cells else 0.0 + ) + successful_tasks_by_policy: dict[str, set[str]] = {} + for cell in successful_cells: + successful_tasks_by_policy.setdefault(cell["policy_name"], set()).add( + cell["task_id"] + ) + paired_task_ids = successful_tasks_by_policy.get("route_once", set()) & ( + successful_tasks_by_policy.get("conduct_bounded", set()) + ) + sufficient = ( + locked_task_count >= MINIMUM_PAIRED_TASK_COUNT + and len(paired_task_ids) >= MINIMUM_PAIRED_TASK_COUNT + and completion_fraction >= REQUIRED_COMPLETION_FRACTION + ) + return { + "evidence_status": ( + "evidence_review_required" if sufficient else "insufficient_evidence" + ), + "decision_use": ( + "production_candidate_review" if sufficient else "benchmark_smoke_only" + ), + "minimum_paired_task_count": MINIMUM_PAIRED_TASK_COUNT, + "required_completion_fraction": REQUIRED_COMPLETION_FRACTION, + "observed_locked_task_count": locked_task_count, + "observed_paired_task_count": len(paired_task_ids), + "observed_completion_fraction": completion_fraction, + "routing_recommendation": None, + } + + +# -------------------------------------------------------------------------- +# Provenance, report schema, artifacts +# -------------------------------------------------------------------------- + + +def sha256_of_file(path: str) -> str: + """Hex SHA-256 of a file's bytes (manifest/pricing provenance hashes).""" + with open(path, "rb") as handle: + return hashlib.sha256(handle.read()).hexdigest() + + +def sha256_of_json(value: Any) -> str: + """Hex SHA-256 of a canonical JSON serialization (catalog snapshot hash).""" + return hashlib.sha256( + json.dumps(value, sort_keys=True, ensure_ascii=False).encode("utf-8") + ).hexdigest() + + +def build_provenance( + run_mode: str, + git_sha: str, + workflow_run_id: str, + catalog_snapshot: dict[str, Any], + task_manifest_path: str, + pricing_scenario_path: str | None, + benchmark_parameters: dict[str, Any], +) -> dict[str, Any]: + """Assemble the provenance block; live runs fail closed on missing identity.""" + if run_mode == "live": + _validate_live_provenance(git_sha, workflow_run_id) + return { + "run_mode": run_mode, + "git_sha": git_sha or DRY_RUN_PROVENANCE_PLACEHOLDER, + "workflow_run_id": workflow_run_id or DRY_RUN_PROVENANCE_PLACEHOLDER, + "catalog_snapshot_sha256": sha256_of_json(catalog_snapshot), + "task_manifest_sha256": sha256_of_file(task_manifest_path), + "pricing_scenario_sha256": ( + sha256_of_file(pricing_scenario_path) if pricing_scenario_path else None + ), + "benchmark_parameters": benchmark_parameters, + } + + +def _validate_live_provenance(git_sha: str, workflow_run_id: str) -> None: + """Reject live evidence that cannot identify an exact Git revision.""" + if ( + re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", git_sha) is None + or not workflow_run_id.strip() + ): + raise BenchmarkContractError( + "live runs require a valid --git-sha and --workflow-run-id provenance" + ) + + +_REPORT_REQUIRED_PATHS = ( + "benchmark_schema_version", + "provenance.run_mode", + "provenance.git_sha", + "provenance.workflow_run_id", + "provenance.catalog_snapshot_sha256", + "provenance.task_manifest_sha256", + "provenance.benchmark_parameters", + "catalog_snapshot.endpoint", + "catalog_snapshot.discovered_model_count", + "catalog_snapshot.duplicate_model_ids", + "catalog_snapshot.invalid_entries", + "catalog_snapshot.probed_models", + "capability_summary", + "evaluation.evaluation_cells", + "evaluation.policy_summaries", + "evaluation.paired_comparisons", + "evaluation.pareto_frontiers", + "evaluation.evidence_status", + "evaluation.decision_use", + "evaluation.minimum_paired_task_count", + "evaluation.required_completion_fraction", + "evaluation.observed_paired_task_count", + "evaluation.observed_completion_fraction", + "evaluation.routing_recommendation", + "request_budget.max_total_requests", + "request_budget.requests_spent", + "request_budget.planned_total_requests", + "request_budget.catalog_requests", + "request_budget.capability_probe_requests", + "request_budget.evaluation_reserve_requests", + "request_budget.planned_worker_count", + "actual_cost_evidence", + "honesty_labels.actual_cost_basis", + "honesty_labels.provider_latency_source", + "honesty_labels.hypothetical_cost_source", +) + + +def validate_report_schema(report: dict[str, Any]) -> None: + """Fail closed when any required report path is absent.""" + missing = [] + for path in _REPORT_REQUIRED_PATHS: + node: Any = report + for key in path.split("."): + if not isinstance(node, dict) or key not in node: + missing.append(path) + break + node = node[key] + if missing: + raise BenchmarkContractError( + f"benchmark report is missing required paths: {missing}" + ) + + +_CSV_CELL_COLUMNS = ( + "policy_name", + "task_id", + "task_split", + "scorer_name", + "scorer_version", + "task_score", + "run_outcome", + "outcome_reason", + "end_to_end_latency_ms", + "provider_latency_ms", + "call_count", + "workflow_depth", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "token_usage_source", + "configured_total_token_budget", + "configured_maximum_calls", + "observed_budget_tokens", + "observed_budget_calls", + "remaining_budget_tokens", + "actual_cost_usd", + "hypothetical_cost_usd", + "response_sha256", +) + + +def _ensure_secret_absent(serialized: str) -> None: + """Refuse to write any artifact that contains the resolved provider secret.""" + secret = get_credential(NIM_CREDENTIAL_NAME) + if secret and secret in serialized: + raise SecretLeakError( + "benchmark artifact would contain the provider credential; refusing to write" + ) + + +def _safe_failure_message(exc: Exception) -> str: + """Return a bounded diagnostic with the resolved benchmark credential removed.""" + message = redact_text(str(exc)) + secret = get_credential(NIM_CREDENTIAL_NAME) + if secret: + message = message.replace(secret, "[REDACTED]") + return message[:500] + + +def render_markdown_summary(report: dict[str, Any]) -> str: + """Render a buyer-readable summary with evidence and cost caveats.""" + lines = [ + "# NIM cost-quality benchmark summary", + "", + f"- run mode: `{report['provenance']['run_mode']}`", + f"- git sha: `{report['provenance']['git_sha']}`", + f"- workflow run id: `{report['provenance']['workflow_run_id']}`", + f"- catalog snapshot sha256: `{report['provenance']['catalog_snapshot_sha256']}`", + f"- discovered models: {report['catalog_snapshot']['discovered_model_count']}", + f"- requests spent: {report['request_budget']['requests_spent']}" + f" / {report['request_budget']['max_total_requests']}", + f"- complete request plan: {report['request_budget']['planned_total_requests']} " + "(catalog + all capability probes + evaluation reserve)", + f"- evidence status: `{report['evaluation']['evidence_status']}`", + f"- decision use: `{report['evaluation']['decision_use']}`", + "", + "## Capability classifications", + "", + "| classification | models |", + "| --- | --- |", + ] + for classification, count in sorted(report["capability_summary"].items()): + lines.append(f"| {classification} | {count} |") + lines += [ + "", + "## Policy summaries (locked split)", + "", + "| policy | mean score | mean latency ms | mean hypothetical cost USD | actual cost USD |", + "| --- | --- | --- | --- | --- |", + ] + for row in report["evaluation"]["policy_summaries"]: + lines.append( + f"| {row['policy_name']} | {row['mean_task_score']} | " + f"{row['mean_latency_ms']} | {row['mean_hypothetical_cost_usd']} " + f"| {row['actual_cost_usd']} |" + ) + lines += ["", "## Paired comparisons (95% bootstrap CI)", ""] + for comparison in report["evaluation"]["paired_comparisons"]: + lines.append( + f"- `{comparison['policy_a']}` vs `{comparison['policy_b']}`: " + f"mean diff {comparison['mean_difference']} " + f"[{comparison['ci_low']}, {comparison['ci_high']}]" + ) + evidence = report["actual_cost_evidence"] + lines += [ + "", + "## Evidence sufficiency", + "", + f"- paired tasks: {report['evaluation']['observed_paired_task_count']} " + f"/ {report['evaluation']['minimum_paired_task_count']} required", + f"- completion fraction: {report['evaluation']['observed_completion_fraction']} " + f"/ {report['evaluation']['required_completion_fraction']} required", + "- production routing recommendation: none" + if report["evaluation"]["routing_recommendation"] is None + else f"- production routing recommendation: {report['evaluation']['routing_recommendation']}", + "", + "## Actual API access-cost evidence", + "", + f"- source: {evidence['source_title']}", + f"- reviewed: {evidence['reviewed_at_date']}", + f"- valid until: {evidence['valid_until_date']}", + f"- access context: {evidence['access_program']} — {evidence['access_scope']}", + f"- production distinction: {evidence['production_access_note']}", + f"- uncertainty: {evidence['uncertainty']}", + "", + "## Honesty labels", + "", + f"- actual cost basis: {report['honesty_labels']['actual_cost_basis']}", + f"- provider latency: {report['honesty_labels']['provider_latency_source']}", + f"- hypothetical cost source: {report['honesty_labels']['hypothetical_cost_source']}", + "", + ] + return "\n".join(lines) + + +def write_benchmark_artifacts( + report: dict[str, Any], + output_dir: str, +) -> dict[str, str]: + """Validate and atomically publish the shared four-artifact evidence set.""" + _validate_actual_cost_evidence(report) + validate_report_schema(report) + json_text = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + _ensure_secret_absent(json_text) + csv_buffer = io.StringIO() + writer = csv.DictWriter( + csv_buffer, + fieldnames=_CSV_CELL_COLUMNS, + extrasaction="ignore", + ) + writer.writeheader() + for cell in report["evaluation"]["evaluation_cells"]: + writer.writerow(cell) + csv_text = csv_buffer.getvalue() + _ensure_secret_absent(csv_text) + + markdown_text = render_markdown_summary(report) + _ensure_secret_absent(markdown_text) + provenance = report["provenance"] + shared_provenance = { + "source_commit": ( + provenance["git_sha"] + if re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", provenance["git_sha"]) + else "0" * 40 + ), + "catalog_snapshot_sha256": provenance["catalog_snapshot_sha256"], + "task_manifest_sha256": provenance["task_manifest_sha256"], + "pricing_scenario_sha256": provenance["pricing_scenario_sha256"] or "unknown", + "workflow_run_id": provenance["workflow_run_id"], + "evidence_status": report["evaluation"]["evidence_status"], + } + provenance_text = json.dumps( + shared_provenance, ensure_ascii=False, indent=2, sort_keys=True + ) + _ensure_secret_absent(provenance_text) + publish_artifact_set( + output_dir, + { + "benchmark_report.json": (json_text + "\n").encode(), + "benchmark_cells.csv": csv_text.encode(), + "benchmark_summary.md": (markdown_text + "\n").encode(), + "run_provenance.json": (provenance_text + "\n").encode(), + }, + ) + return { + "json_path": os.path.join(output_dir, "benchmark_report.json"), + "csv_path": os.path.join(output_dir, "benchmark_cells.csv"), + "markdown_path": os.path.join(output_dir, "benchmark_summary.md"), + "provenance_path": os.path.join(output_dir, "run_provenance.json"), + } + + +# -------------------------------------------------------------------------- +# Benchmark assembly (shared by dry and live runs) +# -------------------------------------------------------------------------- + + +def assemble_benchmark_report( + run_mode: str, + endpoint: str, + catalog: dict[str, Any], + probed_models: list[dict[str, Any]], + evaluation: dict[str, Any], + request_budget: RequestBudget, + provenance_inputs: dict[str, Any], + seed: int, +) -> dict[str, Any]: + """Assemble and validate the complete evidence-grade benchmark report.""" + cells = evaluation["evaluation_cells"] + summaries = summarize_policies(cells) + capability_summary: dict[str, int] = {} + for row in probed_models: + capability_summary[row["model_classification"]] = ( + capability_summary.get(row["model_classification"], 0) + 1 + ) + catalog_snapshot = { + "endpoint": endpoint, + "discovered_model_count": len(catalog["models"]), + "duplicate_model_ids": catalog["duplicate_model_ids"], + "invalid_entries": catalog["invalid_entries"], + "probed_models": probed_models, + } + evidence_summary = _evaluation_evidence_summary( + cells, + evaluation["locked_task_count"], + ) + report = { + "benchmark_schema_version": BENCHMARK_SCHEMA_VERSION, + "provenance": build_provenance( + run_mode, + provenance_inputs["git_sha"], + provenance_inputs["workflow_run_id"], + catalog_snapshot, + provenance_inputs["task_manifest_path"], + provenance_inputs["pricing_scenario_path"], + provenance_inputs["benchmark_parameters"], + ), + "catalog_snapshot": catalog_snapshot, + "capability_summary": capability_summary, + "evaluation": { + "evaluation_cells": cells, + "policy_summaries": summaries, + "best_single_worker_hindsight": best_single_worker_hindsight(summaries), + "paired_comparisons": paired_policy_comparisons(cells, seed=seed), + "pareto_frontiers": build_pareto_frontiers(summaries), + "cheapest_worker_skip_reason": evaluation["cheapest_worker_skip_reason"], + "locked_task_count": evaluation["locked_task_count"], + "worker_count": evaluation["worker_count"], + **evidence_summary, + }, + "request_budget": { + "max_total_requests": request_budget.max_total_requests, + "requests_spent": request_budget.requests_spent, + "planned_total_requests": provenance_inputs["request_plan"][ + "total_required_request_count" + ], + "catalog_requests": provenance_inputs["request_plan"][ + "catalog_request_count" + ], + "capability_probe_requests": provenance_inputs["request_plan"][ + "capability_probe_request_count" + ], + "evaluation_reserve_requests": provenance_inputs["request_plan"][ + "evaluation_reserve_request_count" + ], + "planned_worker_count": provenance_inputs["request_plan"][ + "planned_worker_count" + ], + }, + "actual_cost_evidence": dict(ACTUAL_COST_EVIDENCE), + "honesty_labels": { + "actual_cost_basis": ( + "deterministic_dry_run_no_provider_egress" + if run_mode == "dry_run" + else "reviewed_nvidia_developer_program_hosted_endpoint_access" + ), + "provider_latency_source": ("not_observable_via_openai_compatible_body"), + "hypothetical_cost_source": ( + "explicit_versioned_pricing_scenario_or_unknown" + ), + "dry_run_scores_note": ( + "dry-run scores reflect deterministic mock echoes, not model quality" + ), + }, + } + _validate_actual_cost_evidence(report) + validate_report_schema(report) + return report + + +# -------------------------------------------------------------------------- +# Deterministic dry-run provider (all modality classes, no network) +# -------------------------------------------------------------------------- + +# Synthetic catalog covering every capability class the harness can emit, +# plus one duplicate id and one invalid entry to exercise catalog hygiene. +_DRY_RUN_MODEL_BEHAVIOR = { + "dryrun/chat-basic": {"chat_completion"}, + "dryrun/chat-vision": {"chat_completion", "image_understanding"}, + "dryrun/chat-omni": { + "chat_completion", + "image_understanding", + "video_understanding", + "audio_understanding", + }, + "dryrun/chat-video": {"chat_completion", "video_understanding"}, + "dryrun/embed-basic": {"text_embedding"}, + "dryrun/completion-legacy": {"text_completion"}, + "dryrun/responses-native": {"response_generation"}, + "dryrun/audio-transcribe": {"audio_transcription"}, + "dryrun/audio-speech": {"audio_speech"}, + "dryrun/throttled-model": "rate_limited", + "dryrun/outage-model": "unavailable", + "dryrun/legacy-unsupported": "unsupported", +} + + +def _dry_run_catalog_body() -> bytes: + """Serialized synthetic /v1/models body, including hygiene edge cases.""" + data = [ + {"id": model_id, "owned_by": "dryrun"} for model_id in _DRY_RUN_MODEL_BEHAVIOR + ] + data.append({"id": "dryrun/chat-basic", "owned_by": "dryrun"}) # duplicate id + data.append({"owned_by": "dryrun"}) # missing model id + return json.dumps({"object": "list", "data": data}).encode("utf-8") + + +def _dry_run_success_body(path: str) -> bytes: + """Minimal valid success body for each probed endpoint contract.""" + if path.endswith("/embeddings"): + return json.dumps({"data": [{"embedding": [0.0, 0.1]}]}).encode("utf-8") + if path.endswith("/responses"): + return json.dumps({"output_text": "OK"}).encode("utf-8") + if path.endswith("/audio/transcriptions"): + return json.dumps({"text": "ok"}).encode("utf-8") + if path.endswith("/audio/speech"): + return b"RIFF\x00\x00\x00\x00WAVEdryrunaudio" + return json.dumps({"choices": [{"message": {"content": "OK"}}]}).encode("utf-8") + + +def _dry_run_probe_capability(path: str, body: bytes | None) -> str: + """Infer which capability a dry-run probe request represents.""" + if path.endswith("/chat/completions"): + text = (body or b"").decode("utf-8") + if "image_url" in text: + return "image_understanding" + if "video_url" in text: + return "video_understanding" + if "input_audio" in text: + return "audio_understanding" + return "chat_completion" + for capability_name, spec in CAPABILITY_PROBES.items(): + if path.endswith(spec["path"]) and capability_name != "chat_completion": + return capability_name + raise CatalogDiscoveryError( + f"dry-run transport received an unexpected path: {path}" + ) + + +def build_dry_run_transport() -> ProviderTransport: + """In-process provider fake serving the synthetic all-modality catalog.""" + + def transport( + method: str, url: str, headers: dict[str, str], body: bytes | None + ) -> tuple[int, bytes]: + """Serve catalog and probe requests deterministically without network.""" + path = urllib.parse.urlparse(url).path + if method == "GET" and path.endswith("/models"): + return 200, _dry_run_catalog_body() + model_match = re.search(rb'"model"\s*:\s*"([^"]+)"', body or b"") + if model_match is None: + model_match = re.search(rb'name="model"\r\n\r\n([^\r]+)', body or b"") + model_id = model_match.group(1).decode("utf-8") if model_match else "" + behavior = _DRY_RUN_MODEL_BEHAVIOR.get(model_id) + if behavior is None: + return 404, json.dumps({"error": "unknown dry-run model"}).encode("utf-8") + if behavior == "rate_limited": + return 429, b"{}" + if behavior == "unavailable": + return 503, b"{}" + if behavior == "unsupported": + return 404, b"{}" + capability_name = _dry_run_probe_capability(path, body) + if capability_name in behavior: + return 200, _dry_run_success_body(path) + return 400, json.dumps({"error": "capability not supported"}).encode("utf-8") + + return transport + + +def _deterministic_timer() -> Callable[[], float]: + """Monotonic fake timer for reproducible dry-run latency fields.""" + state = {"now": 0.0} + + def timer() -> float: + """Advance one millisecond per observation.""" + state["now"] += 0.001 + return state["now"] + + return timer + + +# -------------------------------------------------------------------------- +# Run orchestration + CLI +# -------------------------------------------------------------------------- + + +def run_benchmark( + run_mode: str, + task_manifest_path: str, + pricing_scenario_path: str | None, + output_dir: str, + endpoint: str = NIM_DEFAULT_ENDPOINT, + max_total_requests: int = 2000, + probe_concurrency: int = 4, + timeout_seconds: float = 60.0, + max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS, + max_eval_models: int = 7, + seed: int = 7, + git_sha: str = "", + workflow_run_id: str = "", + transport: ProviderTransport | None = None, +) -> dict[str, Any]: + """Run the complete benchmark in deterministic dry or evidence-gated live mode. + + Live evidence, optional paid-price provenance, and run identity are validated + before any provider transport can execute. Dry runs use an in-process provider + and never need or read the NVIDIA credential. + + Args: + run_mode: ``dry_run`` or ``live``. + task_manifest_path: Versioned locked/exploratory task manifest. + pricing_scenario_path: Optional explicit hypothetical pricing scenario. + output_dir: Destination for JSON, CSV, and Markdown artifacts. + endpoint: OpenAI-compatible provider endpoint. + max_total_requests: Complete-run provider request cap. + probe_concurrency: Maximum concurrent model probe workers. + timeout_seconds: Per-address network timeout. + max_output_tokens: Per-provider-call output-token cap. The equal + cell-wide prompt-plus-completion budget is this value multiplied + by ``MAX_WORKFLOW_DEPTH``. + max_eval_models: Maximum chat-eligible workers in policy evaluation. + seed: Deterministic bootstrap seed. + git_sha: Exact source revision, required live. + workflow_run_id: Workflow provenance identifier, required live. + transport: Optional injected provider transport for deterministic tests. + + Returns: + Complete report including written artifact paths. + + Raises: + BenchmarkContractError: If mode, evidence, or parameters are invalid. + NotConfigured: If a live run cannot resolve its KV credential. + """ + if run_mode not in ("dry_run", "live"): + raise BenchmarkContractError( + f"run_mode must be 'dry_run' or 'live', not {run_mode!r}" + ) + if ( + isinstance(max_output_tokens, bool) + or not isinstance(max_output_tokens, int) + or max_output_tokens < 1 + ): + raise BenchmarkContractError("max_output_tokens must be a positive integer") + manifest = load_task_manifest(task_manifest_path) + pricing_scenario = load_pricing_scenario(pricing_scenario_path) + if run_mode == "live": + _validate_live_provenance(git_sha, workflow_run_id) + _require_current_actual_cost_evidence() + validate_live_pricing_scenario(pricing_scenario) + request_budget = RequestBudget(max_total_requests) + + if run_mode == "dry_run": + api_key = "dry-run-placeholder-not-a-secret" + active_transport = transport or build_dry_run_transport() + + def dry_run_clock() -> float: + """Return the fixed timestamp used by deterministic dry runs.""" + return DRY_RUN_FIXED_UNIX_TIME + + def dry_run_probe_timer() -> float: + """Return a zero-duration probe clock for deterministic evidence.""" + return 0.0 + + clock: Callable[[], float] = dry_run_clock + probe_timer: Callable[[], float] = dry_run_probe_timer + timer = _deterministic_timer() + eval_base_url = "mock://nim-dry-run" + eval_client: ModelClient = _BudgetedModelClient( + request_budget, + transport=active_transport, + max_output_tokens=max_output_tokens, + ) + else: + api_key = get_credential(NIM_CREDENTIAL_NAME) or "" + if not api_key: + raise NotConfigured( + f"live benchmark requires the '{NIM_CREDENTIAL_NAME}' credential " + "in the KV; seed it via register-credential bootstrap (never argv)" + ) + active_transport = transport or build_default_transport(timeout_seconds) + clock = time.time + probe_timer = time.perf_counter + timer = time.perf_counter + eval_base_url = endpoint + eval_client = _BudgetedModelClient( + request_budget, + transport=active_transport, + timeout=float(timeout_seconds), + max_output_tokens=max_output_tokens, + ) + + benchmark_parameters = { + "endpoint": endpoint, + "max_total_requests": max_total_requests, + "probe_concurrency": probe_concurrency, + "timeout_seconds": timeout_seconds, + "max_output_tokens": max_output_tokens, + "max_eval_models": max_eval_models, + "max_workflow_depth": MAX_WORKFLOW_DEPTH, + "policy_total_token_budget": max_output_tokens * MAX_WORKFLOW_DEPTH, + "policy_maximum_calls": MAX_WORKFLOW_DEPTH, + "minimum_paired_task_count": MINIMUM_PAIRED_TASK_COUNT, + "required_completion_fraction": REQUIRED_COMPLETION_FRACTION, + "seed": seed, + "task_manifest_version": manifest["manifest_version"], + "pricing_scenario_version": ( + pricing_scenario["scenario_version"] if pricing_scenario else None + ), + "pricing_scenario_status": ( + pricing_scenario["scenario_status"] if pricing_scenario else None + ), + } + + catalog = discover_model_catalog( + active_transport, + endpoint, + api_key, + request_budget, + ) + request_plan = plan_complete_request_budget( + discovered_model_count=len(catalog["models"]), + max_eval_models=max_eval_models, + locked_task_count=len(locked_evaluation_tasks(manifest)), + ) + if request_plan["total_required_request_count"] > request_budget.max_total_requests: + raise BenchmarkBudgetError( + "complete benchmark needs " + f"{request_plan['total_required_request_count']} requests but " + f"configured cap is {request_budget.max_total_requests}; " + "no capability probes were started" + ) + benchmark_parameters.update( + { + "catalog_request_count": request_plan["catalog_request_count"], + "capability_probe_request_count": request_plan[ + "capability_probe_request_count" + ], + "evaluation_reserve_request_count": request_plan[ + "evaluation_reserve_request_count" + ], + "planned_worker_count": request_plan["planned_worker_count"], + "total_required_request_count": request_plan[ + "total_required_request_count" + ], + } + ) + probed_models = probe_discovered_models( + catalog["models"], + active_transport, + endpoint, + api_key, + request_budget, + probe_concurrency, + clock, + probe_timer, + ) + agents = build_worker_agents(probed_models, eval_base_url, max_eval_models) + evaluation = evaluate_policies( + agents, + manifest, + pricing_scenario, + eval_client, + request_budget, + timer, + total_token_budget=max_output_tokens * MAX_WORKFLOW_DEPTH, + maximum_calls=MAX_WORKFLOW_DEPTH, + ) + report = assemble_benchmark_report( + run_mode, + endpoint, + catalog, + probed_models, + evaluation, + request_budget, + { + "git_sha": git_sha, + "workflow_run_id": workflow_run_id, + "task_manifest_path": task_manifest_path, + "pricing_scenario_path": pricing_scenario_path, + "benchmark_parameters": benchmark_parameters, + "request_plan": request_plan, + }, + seed, + ) + report["artifact_paths"] = write_benchmark_artifacts(report, output_dir) + return report + + +def _bootstrap_live_credential() -> None: + """One-shot bootstrap: move the job-environment secret into the KV. + + Environment is used strictly as bootstrap transport (the same contract as + ``register-credential --from-env``); runtime reads then resolve the key + through :func:`get_credential` only. + """ + if get_credential(NIM_CREDENTIAL_NAME) is None and os.environ.get( + NIM_CREDENTIAL_NAME + ): + register_credential(NIM_CREDENTIAL_NAME, os.environ[NIM_CREDENTIAL_NAME]) + + +def run_benchmark_cli(argv: list[str]) -> int: + """CLI entry for ``python -m contextual_orchestrator nim-benchmark``. + + The provider secret is never accepted via argv: live runs resolve it from + the KV, seeded from the job environment by the one-shot bootstrap step. + """ + parser = argparse.ArgumentParser( + prog="python -m contextual_orchestrator nim-benchmark", + description="Evidence-grade NVIDIA NIM model discovery and cost-quality benchmark.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Validate everything without contacting NVIDIA.", + ) + parser.add_argument("--task-manifest", default="examples/nim_task_manifest.json") + parser.add_argument( + "--pricing-scenario", + default=None, + help="Versioned hypothetical price-assumption JSON (omit => costs stay 'unknown').", + ) + parser.add_argument("--output-dir", default="benchmark_artifacts") + parser.add_argument("--endpoint", default=NIM_DEFAULT_ENDPOINT) + parser.add_argument("--max-total-requests", type=int, default=2000) + parser.add_argument("--probe-concurrency", type=int, default=4) + parser.add_argument("--timeout-seconds", type=float, default=60.0) + parser.add_argument( + "--max-output-tokens", type=int, default=DEFAULT_MAX_OUTPUT_TOKENS + ) + parser.add_argument("--max-eval-models", type=int, default=7) + parser.add_argument("--seed", type=int, default=7) + parser.add_argument( + "--git-sha", + default="", + help="Provenance: the exact commit under benchmark (required live).", + ) + parser.add_argument( + "--workflow-run-id", + default="", + help="Provenance: the CI run id (required live).", + ) + args = parser.parse_args(argv) + + run_mode = "dry_run" if args.dry_run else "live" + if run_mode == "live": + _bootstrap_live_credential() + try: + report = run_benchmark( + run_mode, + args.task_manifest, + args.pricing_scenario, + args.output_dir, + endpoint=args.endpoint, + max_total_requests=args.max_total_requests, + probe_concurrency=args.probe_concurrency, + timeout_seconds=args.timeout_seconds, + max_output_tokens=args.max_output_tokens, + max_eval_models=args.max_eval_models, + seed=args.seed, + git_sha=args.git_sha, + workflow_run_id=args.workflow_run_id, + ) + except ( + BenchmarkContractError, + CatalogDiscoveryError, + BenchmarkAuthError, + BenchmarkBudgetError, + SecretLeakError, + NotConfigured, + OSError, + ) as exc: + print( + json.dumps( + { + "benchmark_failed_closed": True, + "error_class": type(exc).__name__, + "error_message": _safe_failure_message(exc), + }, + ensure_ascii=False, + ) + ) + return 1 + print( + json.dumps( + { + "run_mode": report["provenance"]["run_mode"], + "discovered_model_count": report["catalog_snapshot"][ + "discovered_model_count" + ], + "capability_summary": report["capability_summary"], + "requests_spent": report["request_budget"]["requests_spent"], + "artifact_paths": report["artifact_paths"], + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 diff --git a/contextual_orchestrator/provider_transport.py b/contextual_orchestrator/provider_transport.py new file mode 100644 index 000000000..aff96b087 --- /dev/null +++ b/contextual_orchestrator/provider_transport.py @@ -0,0 +1,70 @@ +"""DNS-pinned HTTPS primitives for validated model-provider egress. + +``ModelClient`` owns policy validation and request dispatch directly. This +module contains only focused connection, response-cleanup, and public-address +validation helpers, so importing the package never mutates another class. +""" + +from __future__ import annotations + +import http.client +import ipaddress +import socket +import ssl + + +class PinnedHTTPSConnection(http.client.HTTPSConnection): + """Connect to one validated IP while retaining the provider hostname for TLS.""" + + def __init__( + self, + server_hostname: str, + pinned_ip: str, + port: int, + timeout: float, + context: ssl.SSLContext, + ) -> None: + """Configure a direct TLS connection to a previously validated address.""" + super().__init__(server_hostname, port=port, timeout=timeout, context=context) + self._pinned_ip = pinned_ip + self._server_hostname = server_hostname + + def connect(self) -> None: + """Dial the pinned IP and verify the certificate against the original host.""" + raw_socket = socket.create_connection( + (self._pinned_ip, self.port), + self.timeout, + self.source_address, + ) + try: + self.sock = self._context.wrap_socket( + raw_socket, + server_hostname=self._server_hostname, + ) + except Exception: # noqa: BLE001 - close the raw socket, then preserve the TLS failure. + raw_socket.close() + raise + + +def validated_public_addresses( + hostname: str, port: int, provider_label: str +) -> tuple[str, ...]: + """Resolve, validate, and deduplicate addresses approved for one connection.""" + validated_addresses: list[str] = [] + for address in socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM): + resolved_address = ipaddress.ip_address(address[4][0]) + if ( + not resolved_address.is_global + or resolved_address.is_private + or resolved_address.is_loopback + or resolved_address.is_link_local + or resolved_address.is_multicast + or resolved_address.is_reserved + ): + raise RuntimeError(f"{provider_label} provider resolves to non-public address") + normalized_address = str(resolved_address) + if normalized_address not in validated_addresses: + validated_addresses.append(normalized_address) + if not validated_addresses: + raise RuntimeError(f"{provider_label} provider host did not resolve") + return tuple(validated_addresses) diff --git a/docs/architecture.md b/docs/architecture.md index 2de247254..2ef49f49c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -119,6 +119,7 @@ the worker is read, patched, or removed. The deliberate simplification is the policy. The paper systems learn routing and topology from rewards; this lab uses a deterministic capability-hint heuristic only for worker/role routing so the repo runs without training data, GPUs, or vendor credentials. It is never an answer-quality, verification, or accept/reject judgment: verifier decisions must use the structured model judge and fail closed (see [ADR 0001](planning/adrs/0001-fail-closed-model-judgment.md)). Add learned routing only when there is an evaluation set and logs proving the heuristic policy is the bottleneck. +The [NIM cost-quality benchmark](nim_benchmark.md) is that evaluation set's supplier: it discovers the hosted catalog dynamically, probes every modality contract, and compares route/conduct/single-worker policies with paired uncertainty — evidence first, learned policy later. ## SDK omit-real persist diff --git a/docs/doctoring/nim-benchmark-evidence-grade.md b/docs/doctoring/nim-benchmark-evidence-grade.md new file mode 100644 index 000000000..14a197491 --- /dev/null +++ b/docs/doctoring/nim-benchmark-evidence-grade.md @@ -0,0 +1,252 @@ +# Evidence-grade NVIDIA NIM benchmark: engineering decision record + +## Decision + +The NVIDIA NIM benchmark is an optional, provider-neutral evaluation adapter. +It is not imported by the normal package initializer and it never modifies the +runtime gateway as an import side effect. Live execution uses the same +validation-time-address-pinned HTTPS boundary as the gateway, while dry +execution remains deterministic, network-free, and credential-free. + +The benchmark is evidence-generating rather than policy-authorizing. It records +what was discovered, planned, attempted, completed, failed, measured, estimated, +and unknown. It never changes production routing automatically. A report below the +explicit evidence floor is labeled `insufficient_evidence`, and every report +keeps `routing_recommendation` null so a responsible human review remains +necessary. + +## Architecture and MSA boundary + +`contextual_orchestrator/nim_benchmark.py` owns catalog discovery, capability +probing, equal-budget policy comparison, evidence validity, uncertainty, +Pareto analysis, and artifact serialization. The ordinary gateway remains +standalone and provider-neutral. Other ContextualWisdomLab services may invoke +the benchmark as a module or CLI without taking ownership of its transport, +credential, pricing, or evidence rules. + +The boundary preserves the following responsibilities: + +- the host workflow owns GitHub Secret delivery and immutable run provenance; +- the benchmark moves the secret into the process-local credential registry and + resolves it by the `NVIDIA_NIM_API_KEY` credential name; +- the benchmark owns bounded provider calls and secret-redacted artifacts; +- the central `.github` repository owns independent review and protected-branch + policy; and +- consumers such as naruon may read artifacts but do not receive authority to + reinterpret unknown prices or incomplete evidence as production facts. + +## Provider-egress security contract + +A conventional URL opener is not used for live NIM requests. Validation and +connection are one security boundary: + +1. Parse an HTTPS URL and reject missing hostnames. +2. Resolve the hostname once for that request. +3. Reject any answer that is not globally routable, including private, + loopback, link-local, multicast, reserved, unspecified, IPv6 unique-local, + and RFC 6598 shared address space. +4. Dial only an address from that exact validation result. +5. Preserve the original hostname for HTTP authority, TLS SNI, and certificate + hostname verification. +6. Do not consult environment proxy settings. +7. Reject every redirect before a bearer credential can reach another origin. +8. Close responses and connections deterministically and use only another + address from the same validation result for fallback. +9. Read at most 8 MiB plus one sentinel byte from a provider response and fail + closed before an oversized body can be materialized into benchmark evidence. + +RFC 6598 defines `100.64.0.0/10` as shared, non-globally-routable address space. +RFC 4193 defines IPv6 unique-local addresses as local rather than globally +routable. RFC 9110 defines redirects as new target-URI actions and identifies +`Authorization` as resource-specific credential material that merits removal +when redirecting. The implementation chooses the narrower fail-closed policy of +not following redirects at all. + +## Dynamic discovery and deterministic probe allocation + +`GET /v1/models` is the run-time source of the model inventory. Model identifiers +are not hard-coded as authoritative catalog entries. The parser records invalid +and duplicate entries and sorts the usable inventory. + +Every discovered model must receive a completed outcome row for every supported +probe contract in a successful live run. Immediately after the catalog request, +the benchmark computes one complete request plan containing the catalog request, +all `(model_id, capability_name)` probes, and the worst-case equal-budget policy +evaluation reserve. If the configured hard cap is even one request short, the +run fails closed before the first capability probe; a lexicographic model prefix +can never be emitted as routing-readiness evidence. + +The acceptance fixture uses 127 discovered models, nine capability contracts, +seven evaluation workers, and thirty locked tasks. Its complete upper bound is +`1 + (127 × 9) + (30 × (2 × 7 + 5 + 5 + 2)) = 1,924` requests. Direct and +cheapest-worker cells reserve a worker call plus a real-time judge call; +route-once reserves its full equal-call envelope, and conduct reserves its +five-call workflow and judge envelope. The monthly workflow runs on the first +day of each month so the next scheduled run falls inside the current reviewed +evidence window; stale evidence still fails closed. It therefore uses a +reviewed hard ceiling of 2,000 requests, leaving bounded room +for catalog growth while retaining a deterministic cap. If a later catalog no +longer fits, the same preflight reports required and configured counts and makes +zero partial probe calls. Once admitted, all probe cells execute under bounded +concurrency; thread scheduling changes only completion order, never inventory +coverage or evaluation capacity. + +The video-understanding probe contains a deterministic, decodable one-frame H.264 +MP4. Its embedded bytes have SHA-256 +`777dda43b5a15162b68a39aa486d5c70c9994d7fe761742fd00d4e13508983c0`. +Startup validation confirms the ISO Base Media File Format structure, a video +handler, AVC sample entry, 16 × 16 dimensions, one sample, and media data. This +prevents a malformed `ftyp`-only stub from turning a capable video model's valid +rejection into a false unsupported classification. + +## Fair policy comparison + +Direct single-worker, `route_once`, bounded `conduct`, and reviewed +cheapest-worker cells receive the same per-task contract: + +- one locked task and scorer version; +- one equal cell-wide prompt-plus-completion token allowance, set to five times + the per-provider-call output cap by default (`1,320` tokens); +- one five-call maximum envelope; +- one timeout policy; and +- one workflow-depth ceiling. + +Provider retries and orchestration tool retries are disabled inside each +benchmark cell so the declared request budget bounds actual egress and the +measured call envelope remains comparable across policies. + +The token allowance is cell-wide rather than per request. Prompt estimates are +charged before a call, the output cap is reduced to the remaining allowance, +and provider-reported usage replaces the latest estimate when valid. Booleans, +negative values, NaN, and infinities are not accepted as token counts. A deep +policy cannot obtain five times a single-call arm's total token budget merely by +issuing five calls. + +## Cost evidence and price honesty + +Actual access cost and hypothetical production cost are separate fields and +separate evidence classes. + +As reviewed on 2026-08-05, NVIDIA's NIM General FAQ states that NVIDIA Developer +Program members have free access to hosted NIM API endpoints for prototyping. +The same source distinguishes development, testing, research, and evaluation +from production and states that production requires NVIDIA AI Enterprise. The +report therefore records `actual_cost_usd = 0.0` only for the reviewed hosted +endpoint access context, includes the exact source, review date, validity +horizon, program scope, production distinction, and uncertainty, and refuses a +live run after 2026-09-04 until the source is reviewed again. + +No NVIDIA model price is embedded or inferred. A live hypothetical pricing +scenario is optional; absence means `unknown`. If supplied, it must be marked +`reviewed` and include an HTTPS source, reviewer, review date, validity horizon, +rate basis, uncertainty, and explicit input/output rates. Unreviewed, future, +incomplete, or expired evidence fails before network egress. The included +example remains deliberately `example_unreviewed` and is valid only for dry-run +schema testing. + +NVIDIA's offering documentation further distinguishes exploratory/free NIM +availability from NIM Certified, which requires NVIDIA AI Enterprise for +enterprise lifecycle, CVE, support, and compliance expectations. The benchmark +does not convert free prototype access into a claim about production licensing, +support, or per-model production price. + +## Evidence sufficiency and uncertainty + +The bundled thirty-task manifest is an evidence-floor fixture with two exploratory +tasks outside the decision set. It verifies the integration surface but cannot +authorize production routing. The governance floor is: + +- at least 30 locked paired tasks shared by compared policies; and +- at least 90% successful cells across the requested comparison matrix. + +These values are explicit conservative release-governance thresholds, not a +claim of universal statistical sufficiency. The artifact reports the observed +paired-task count, requested thresholds, completion fraction, and whether the +run is `insufficient_evidence` or `evidence_review_required`. Even when the floor +is met, production routing remains a human decision and +`routing_recommendation` stays null. + +Paired bootstrap intervals preserve task pairing and expose uncertainty in mean +score differences. Pareto frontiers show quality against latency and reviewed +hypothetical cost; policies with unknown cost are excluded from that cost +frontier and named explicitly. HELM motivates standardized multi-metric +conditions and visible incompleteness. FrugalGPT and RouteLLM motivate measuring +cost-quality routing trade-offs, but their results are not treated as evidence +for this repository's models or tasks. + +## Workflow and credential separation + +`.github/workflows/nim-benchmark.yml` has separate dry and live jobs. The dry job +never receives `NVIDIA_NIM_API_KEY`. Only the live benchmark step receives the +GitHub Secret, and the credential is never passed through argv or written to an +artifact. Both jobs use immutable action revisions, bounded execution, and +single-flight concurrency. The workflow cannot merge, release, approve its own +changes, or modify routing configuration. + +The ordinary test workflow separately proves: + +- complete focused production statement and branch coverage; +- 100% public docstrings; +- wheel build and clean-environment import; +- no eager optional benchmark import; +- no compatibility monkeypatch module; +- no temporary branch-writing or source-export repair job; and +- no retained one-use transformation payload. + +## Verification contract + +An exact pull-request head is eligible for review only after all of the following +succeed: + +- deterministic unit and adversarial security tests; +- transport tests for DNS rebinding, proxy isolation, redirects, SNI/authority, + address fallback, bounded response bodies, and cleanup; +- complete-plan preflight tests for 127 models, the exact boundary, one request short, zero partial egress, deterministic concurrency, and a valid media fixture; +- live pricing and access-evidence expiry tests that prove failure before egress; +- equal token/call budget tests for every comparison arm; +- evidence-sufficiency, Pareto, provenance, and secret-redaction tests; +- 100% statement and branch coverage for the production benchmark module; +- 100% public docstrings; +- package build, install, and import smoke tests; +- repository Tests, Fuzz, Security, Security Scan, and SAST; and +- independent exact-head review with no unresolved actionable thread. + +No earlier head, local-only result, queued check, or stale approval is accepted as +release evidence. + +## References + +Autio, C., Schwartz, R., Dunietz, J., Jain, S., Stanley, M., Tabassi, E., Hall, +P., & Roberts, K. (2024). *Artificial intelligence risk management framework: +Generative artificial intelligence profile* (NIST AI 600-1). National Institute +of Standards and Technology. https://doi.org/10.6028/NIST.AI.600-1 + +Chen, L., Zaharia, M., & Zou, J. (2023). FrugalGPT: How to use large language +models while reducing cost and improving performance. *arXiv*. +https://doi.org/10.48550/arXiv.2305.05176 + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110; +STD 97). RFC Editor. https://doi.org/10.17487/RFC9110 + +Hinden, R., & Haberman, B. (2005). *Unique local IPv6 unicast addresses* +(RFC 4193). RFC Editor. https://doi.org/10.17487/RFC4193 + +Liang, P., Bommasani, R., Lee, T., Tsipras, D., Soylu, D., Yasunaga, M., Zhang, +Y., Narayanan, D., Wu, Y., Kumar, A., Newman, B., Yuan, B., Yan, B., Zhang, C., +Cosgrove, C., Manning, C. D., Ré, C., Acosta-Navas, D., Hudson, D. A., … Koreeda, +Y. (2023). Holistic evaluation of language models. *Transactions on Machine +Learning Research*. https://doi.org/10.48550/arXiv.2211.09110 + +NVIDIA Corporation. (n.d.). *General FAQ*. NVIDIA NIM Documentation. Retrieved +August 5, 2026, from https://docs.api.nvidia.com/nim/docs/product + +NVIDIA Corporation. (2026, June 4). *NIM offerings*. NVIDIA NIM for Large +Language Models. https://docs.nvidia.com/nim/large-language-models/2.0.5/about-nim-llm/nim-offerings.html + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., Kadous, +M. W., & Stoica, I. (2024). RouteLLM: Learning to route LLMs with preference +data. *arXiv*. https://doi.org/10.48550/arXiv.2406.18665 + +Weil, J., Kuarsingh, V., Donley, C., Liljenstolpe, C., & Azinger, M. (2012). +*IANA-reserved IPv4 prefix for shared address space* (RFC 6598; BCP 153). RFC +Editor. https://doi.org/10.17487/RFC6598 diff --git a/docs/nim_benchmark.md b/docs/nim_benchmark.md new file mode 100644 index 000000000..479afdcb2 --- /dev/null +++ b/docs/nim_benchmark.md @@ -0,0 +1,232 @@ +# NIM model discovery + cost-quality benchmark + +The optional benchmark harness (`contextual_orchestrator/nim_benchmark.py`) +addresses issue #86: generate reproducible evidence about how the repository's +routing policies behave on a **real, dynamically discovered** model pool. +NVIDIA NIM is an evaluation provider, not a runtime dependency. Importing the +normal `contextual_orchestrator` package does not import or mutate the optional +benchmark module. + +The detailed engineering and evidence record is +[`docs/doctoring/nim-benchmark-evidence-grade.md`](doctoring/nim-benchmark-evidence-grade.md). + +## Run it + +```bash +# Deterministic dry run: validates manifests, scorers, budgets, evidence +# sufficiency, and artifact schemas against an in-process provider. It performs +# no network calls and never receives NVIDIA_NIM_API_KEY. +python -m contextual_orchestrator nim-benchmark --dry-run \ + --pricing-scenario examples/nim_pricing_scenario.json \ + --output-dir benchmark_artifacts + +# Live CI run: the workflow injects NVIDIA_NIM_API_KEY only into the live step. +# The process bootstraps it into the credential registry and runtime access +# resolves the credential by name. +python -m contextual_orchestrator nim-benchmark \ + --max-total-requests 2000 \ + --max-output-tokens 264 \ + --git-sha "$GITHUB_SHA" \ + --workflow-run-id "$GITHUB_RUN_ID" +``` + +The provider secret is never accepted through argv, printed, or serialized. +Artifact writing fails closed if the resolved secret appears in any output. + +`--max-output-tokens` is the per-provider-call output cap. The equal +cell-wide prompt-plus-completion budget is five times that cap by default +(`1,320` tokens), which leaves the fixed five-call conduct workflow enough room +for its prompts while keeping the same cell budget for every policy. + +## Provider-egress security boundary + +Catalog discovery, probes, and live policy evaluation use validation-time +address pinning: + +- every provider URL must use HTTPS; +- each request resolves its hostname exactly once; +- every answer must be globally routable, so RFC 6598 shared space, private, + loopback, link-local, multicast, reserved, unspecified, and IPv6 unique-local + addresses are rejected; +- the socket dials only an address from that exact DNS answer; +- HTTP authority, TLS SNI, and certificate hostname verification retain the + original hostname; +- environment proxy settings are not consulted; +- redirects are rejected rather than followed; and +- address fallback is limited to the same validation result; and +- every provider response is read through an 8 MiB hard cap before it can be + materialized in memory. + +This closes the DNS time-of-check/time-of-use gap created by validating a +hostname and then letting a generic URL opener resolve it again. + +## Dynamic catalog and all-modality probes + +The live inventory comes from the OpenAI-compatible `GET /v1/models`; no +hard-coded list is treated as authoritative. The parser deduplicates and sorts +usable identifiers while retaining invalid-entry and duplicate evidence. Zero +usable models fails the run closed. + +Every discovered model receives a row for each contract: + +| Probe | Endpoint or request contract | +| --- | --- | +| `chat_completion` | `POST /chat/completions` | +| `text_completion` | `POST /completions` | +| `response_generation` | `POST /responses` | +| `text_embedding` | `POST /embeddings` | +| `image_understanding` | chat with a tiny PNG `image_url` part | +| `video_understanding` | chat with a validated one-frame MP4 `video_url` part | +| `audio_understanding` | chat with a tiny WAV `input_audio` part | +| `audio_transcription` | `POST /audio/transcriptions` | +| `audio_speech` | `POST /audio/speech` | + +After catalog discovery and before the first capability request, the harness +constructs the complete request plan: one discovery request, every sorted +`(model_id, capability_name)` probe, and the conservative evaluation reserve for +the maximum eligible worker pool. If the configured cap is even one request +short, the run fails closed before capability egress and reports the required +and configured counts. Partial model-major prefixes cannot produce routing +evidence. Once preflight passes, all fixed cells execute under bounded +concurrency; thread scheduling can change completion order but not coverage. + +The monthly schedule runs on the first day of each month and uses a hard ceiling +of 2,000 requests. This places the next scheduled run inside the current +reviewed evidence window; stale access-cost evidence still fails closed. On the +127-model catalog scale observed on 2026-08-05, the current thirty-task, seven-worker +configuration requires 1,924 requests: one catalog request, 1,143 capability +probes, and a 780-request worst-case evaluation reserve. The reserve includes +the full equal-call envelope for route-once cells, the five-call conduct +envelope, and real-time judge calls on direct and cheapest-worker cells. Catalog growth +beyond the ceiling causes a zero-partial-egress preflight failure rather than +silent truncation. + +The embedded video fixture is a deterministic, decodable 16 × 16, one-frame +H.264 MP4. Its bytes are verified against SHA-256 +`777dda43b5a15162b68a39aa486d5c70c9994d7fe761742fd00d4e13508983c0`, and its +container structure, video handler, AVC sample entry, dimensions, sample count, +and media data are validated before use. + +Probe outcomes are `supported`, `unsupported`, `rate_limited`, `timeout`, +`unavailable`, `failed`, `malformed_response`, or `skipped`. HTTP 401 fails the +whole run closed. Model-level classes are derived from observations rather than +model names or marketing metadata. + +## Fair comparison contract + +Every policy × task cell receives the same: + +- locked task and scorer version; +- total prompt-plus-completion token allowance configured by + `--max-output-tokens`; +- five-call maximum envelope; +- timeout policy; and +- five-step workflow-depth ceiling. + +Provider retries and orchestration tool retries are disabled inside the benchmark +cell so the declared request budget bounds actual egress and the measured call +envelope remains comparable across policies. + +The token allowance is cell-wide, not per call. Prompt tokens are charged before +a request, the output cap is reduced to the remaining allowance, and valid +provider-reported usage replaces the latest estimate. A deep `conduct` path +cannot receive five times a direct arm's total token budget merely because it +uses more calls. + +Compared policies are: + +1. one direct baseline per chat-eligible worker; +2. deterministic `route_once`; +3. bounded `conduct_bounded`; and +4. a cheapest eligible worker only when an explicit reviewed pricing scenario + supports that comparison. + +Each cell records configured and observed budgets, score and scorer version, +outcome and reason, latency, depth, usage source, model/role/step assignments, +actual and hypothetical cost fields, and a response SHA-256. + +## Cost honesty and evidence validity + +Actual endpoint access and hypothetical paid cost remain separate evidence +classes. + +As reviewed on 2026-08-05, NVIDIA's current General FAQ states that NVIDIA +Developer Program members have free access to hosted NIM API endpoints for +prototyping. The report records that exact source, review date, validity horizon, +program context, production distinction, and uncertainty. A live run fails +closed after 2026-09-04 until the official source is reviewed again. Production +support and licensing are not inferred from prototype access and require +NVIDIA AI Enterprise under the reviewed documentation. + +Hypothetical paid cost is computed only from an explicit pricing scenario. +Omitting a scenario is valid and leaves cost `"unknown"`. A live scenario must +be marked `reviewed` and contain an HTTPS source, reviewer, review date, validity +horizon, rate basis, uncertainty, and explicit input/output rates. Unreviewed, +future, incomplete, or expired scenarios fail before provider egress. The +included example is intentionally `example_unreviewed`; it exists only to test +dry-run schemas and must never be presented as real model pricing. + +## Evidence sufficiency and uncertainty + +The bundled thirty-task manifest is an evidence-floor fixture with two exploratory +tasks kept outside the decision set. It proves integration behavior but does not +authorize production routing. A report reaches +`evidence_review_required` only when it contains at least 30 paired locked tasks +and at least 90% successful comparison cells. Otherwise it reports +`insufficient_evidence` and explains the shortfall. + +These thresholds are explicit conservative governance floors, not universal +statistical guarantees. Every report keeps `routing_recommendation` null even +when the floor is met; a human review remains required. + +- Seeded paired bootstrap intervals preserve task pairing. +- Pareto frontiers cover quality versus latency and quality versus reviewed + hypothetical cost. +- Unknown-cost policies are excluded from the cost frontier and named. +- The manifest rejects expected-answer leakage according to each task's scorer. +- Only locked tasks enter reported comparisons; exploratory tasks remain outside + the decision evidence. + +## Fail-closed contract + +A run aborts without artifacts when live provenance is absent, the credential is +missing, endpoint validation fails, discovery is incomplete, authentication is +rejected, a request/call/token budget is exhausted, cost evidence is invalid or +expired, a provider response exceeds 8 MiB, report validation fails, or output +would contain the provider secret. + +## Provenance and artifacts + +Each run writes: + +- `benchmark_report.json`; +- `benchmark_cells.csv`; and +- `benchmark_summary.md`. + +Provenance records the exact Git SHA and workflow run, catalog/manifest/pricing +hashes, benchmark parameters, capability failures and skips, equal-budget +configuration and observations, access-cost evidence and validity, evidence +sufficiency, uncertainty, and Pareto results. Dry-run artifacts are deterministic +so schema and evidence regressions are reviewable as diffs. + +## Workflow + +`.github/workflows/nim-benchmark.yml` uses separate dry and live jobs. The dry +job has no NVIDIA secret. Only the live benchmark step receives +`NVIDIA_NIM_API_KEY`. Both paths use immutable action revisions, hard request and +execution bounds, single-flight concurrency, and retained artifacts. The +workflow cannot merge, release, approve its own changes, or rewrite production +routing. + +The normal Tests workflow separately proves 100% production statement and +branch coverage, 100% public docstrings, wheel build/install/import behavior, +optional-import isolation, and absence of temporary repair/export mechanisms. + +## Method grounding + +HELM supports standardized multi-metric evaluation and explicit reporting of +coverage gaps. FrugalGPT, RouteLLM, and Hybrid LLM motivate measuring routing +cost-quality trade-offs. NIST AI 600-1 supports documented, risk-aware testing, +evaluation, verification, and validation. These sources shape the measurement +and governance design; they do not substitute for exact-head evidence from the +models, tasks, and policies actually under review. diff --git a/docs/papers/README.md b/docs/papers/README.md index 60fd25ce2..c776afda1 100644 --- a/docs/papers/README.md +++ b/docs/papers/README.md @@ -65,6 +65,19 @@ redistribution is unclear. Buyer next action: call `run_equal_budget_ablation` and read `production_default_change_allowed` before changing live defaults. +## Evaluation methodology (NIM cost-quality benchmark) + +- **Holistic Evaluation of Language Models (HELM)** — Percy Liang, Rishi + Bommasani, Tony Lee, et al. arXiv:2211.09110, 2022 (TMLR 2023). + `helm-holistic-evaluation-2211.09110.pdf` + Grounds the **NIM benchmark harness** (`docs/nim_benchmark.md`): evaluate a + broad, explicitly enumerated model pool on multiple metrics at once (quality, + latency, cost) instead of a single leaderboard number; report incompleteness + honestly (skipped/unsupported/rate-limited cells stay machine-readable rather + than silently dropped); and standardize conditions across compared systems + (same tasks, scorers, caps, and budgets). Distributed under the arXiv +non-exclusive license / CC BY as marked on arXiv. + ## Batch execution / load balancing The external `pg-llm-batch` service carries its own grounding papers, including @@ -76,3 +89,24 @@ but not vendored here so this repository remains one deployable control plane. > Citations are provided for scholarly attribution. Redistribution here relies > on the arXiv non-exclusive distribution license each author granted; no > GPL/AGPL-licensed material is vendored anywhere in this repository. + +## APA 7th edition references + +Chen, L., Zaharia, M., & Zou, J. (2023). FrugalGPT: How to use large language +models while reducing cost and improving performance. *arXiv*. +https://doi.org/10.48550/arXiv.2305.05176 + +Ding, D., Mallick, A., Wang, C., Sim, R., Mukherjee, S., Rühle, V., Lakshmanan, +L. V. S., & Awadallah, A. H. (2024). Hybrid LLM: Cost-efficient and +quality-aware query routing. *arXiv*. +https://doi.org/10.48550/arXiv.2404.14618 + +Liang, P., Bommasani, R., Lee, T., Tsipras, D., Soylu, D., Yasunaga, M., Zhang, +Y., Narayanan, D., Wu, Y., Kumar, A., Newman, B., Yuan, B., Yan, B., Zhang, C., +Cosgrove, C., Manning, C. D., Ré, C., Acosta-Navas, D., Hudson, D. A., … Koreeda, +Y. (2023). Holistic evaluation of language models. *Transactions on Machine +Learning Research*. https://doi.org/10.48550/arXiv.2211.09110 + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., Kadous, +M. W., & Stoica, I. (2024). RouteLLM: Learning to route LLMs with preference +data. *arXiv*. https://doi.org/10.48550/arXiv.2406.18665 diff --git a/docs/papers/helm-holistic-evaluation-2211.09110.pdf b/docs/papers/helm-holistic-evaluation-2211.09110.pdf new file mode 100644 index 000000000..5a6770dea Binary files /dev/null and b/docs/papers/helm-holistic-evaluation-2211.09110.pdf differ diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index cbad42f7f..8953a3a90 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -625,6 +625,48 @@ This document serves as the baseline for the Contextual Orchestrator (an enterpr # Product and Technical Gap Baseline +## 2026-08-30 PR #906 token-budget failure: root-caused and fixed, not flaky + +Two prior passes recorded `tests/test_nim_benchmark_release_acceptance.py:: +test_smoke_manifest_cannot_authorize_production_routing` as failing +(`evidence_status == "insufficient_evidence"`, expected +`"evidence_review_required"`, `configured_total_token_budget=1280` vs +`observed_budget_tokens=1283` on task `trick_arithmetic_lily_pads`/policy +`conduct_bounded`) and both explicitly declined to fix it, guessing it +"depends on live-provider discovery evidence in the hosted runner ... varies +run to run with upstream catalog/availability" and needs "the PR author's +input." That guess is disproven: `dry_run` mode uses +`build_dry_run_transport()` (a fully in-process mock, asserted by the same +test as `actual_cost_basis == "deterministic_dry_run_no_provider_egress"`) +and never touches the network. Re-run in a sandbox with zero live network +access, the failure reproduces byte-for-byte identically every time — +100% deterministic, not flaky. + +Root cause: of the 30 locked tasks, exactly one (`trick_arithmetic_lily_pads`, +whose prompt is slightly longer than its siblings') accumulates enough +JSON-serialized message-history tokens across the four sequential +`conduct_bounded` calls (thinker→worker→verifier→synthesizer) that its +estimated total (1283) exceeds the equal per-cell budget +(`MAX_WORKFLOW_DEPTH(5) * DEFAULT_MAX_OUTPUT_TOKENS(256) = 1280`) by 3 +tokens, tripping `PolicyTokenBudgetExceeded` and flipping that one cell's +`run_outcome` to `"failure"`. That drops the `route_once`/`conduct_bounded` +paired-success count to 29, one below `MINIMUM_PAIRED_TASK_COUNT(30)`, so +`_evaluation_evidence_summary` reports `insufficient_evidence` even though +29 of 30 locked tasks (99.17%) succeeded. The module's own comment states +the intent this violates: the equal-budget envelope should let "a fixed +conduct workflow ... carry its prompts without being starved." A one-task, +3-token-over-a-1280-token-budget margin is exactly that starvation, not a +signal about the manifest or the classification logic. + +Fix: raised `DEFAULT_MAX_OUTPUT_TOKENS` from 256 to 264 (`contextual_orchestrator/nim_benchmark.py`), +giving the derived `DEFAULT_POLICY_TOTAL_TOKEN_BUDGET` (`MAX_WORKFLOW_DEPTH * +DEFAULT_MAX_OUTPUT_TOKENS`, referenced symbolically everywhere it's +asserted) a 40-token margin — comfortably clears the 3-token overage with +headroom for estimator drift, and only affects this optional benchmark +harness's own default, not live orchestration routing/token defaults. All +121 NIM-benchmark tests pass afterward, including this one; 100% +statement/branch coverage and 100% docstrings on `nim_benchmark.py` hold. + ## 2026-08-30 generalize the Models.dev free-cost join beyond opencode_zen `orchestrator/free` (ADR 0032) was structurally empty in practice: `is_free` @@ -766,6 +808,125 @@ A short status comment was left on each of `#868`, `#857`, `#906`, `#911`, and `#912` recording the pin-bump-did-not-fix-it finding so the next pass (human or agent) does not re-diagnose the same sidecar failure from scratch. +## 2026-08-30 hourly loop: #868 test-mock fix, #857 narrow hardening, #906 stale-base merge + +Fresh status check confirmed #868/#911/#912 were still `BLOCKED` purely on the +known org-wide `opencode-review`/`noema-review` failure (stale +`ORCHESTRATOR_PIN_SHA` vendored in `ContextualWisdomLab/.github`, fix pending +in `.github#1422`) — none had picked up an approval since the last pass, so +none were merged this cycle. #911/#912 had no other non-systemic failures +(`Full unit and contract suite` green on both) and needed no code changes. + +**#868** (`fix/gateway-default-chat-model`) had one genuine, non-systemic +failure at the start of this pass: `Full unit and contract suite` failed with +`AttributeError: 'Namespace' object has no attribute 'provider_ca_bundle'` in +`_discover_models_command` (`contextual_orchestrator/__main__.py:305`) — its +own `argparse.ArgumentParser` never declared `--provider-ca-bundle`, even +though the function read `args.provider_ca_bundle` unconditionally (26 tests +failed: 8 directly on the missing attribute, 18 in +`test_auto_discovery_server.py` because their `discover_all_models` mocks +were fixed-arity lambdas that could not accept the `ca_bundle=` keyword the +server-startup call site already passes). Mid-fix, the PR owner +independently pushed `51fc34bb` adding the identical `--provider-ca-bundle` +argument — this pass rebased its own unpushed commit on top of that (no +history rewritten, since the commit had never been shared) and kept only the +non-duplicate half: widening the 18 test lambdas to `**_kwargs`. Pushed as +`e16cfed2`. Full local suite: `2745 passed, 1 skipped, 1 failed` — the one +failure is `tests/test_psychometric_routing.py` needing the private +`fast-mlsirm` package, unreachable in this sandbox (same documented blocker +as PR #917), not a regression. + +**#857** (`fix/provider-backed-embedding-batch`) remains far too diverged to +merge-resolve in one pass (165 files / ~13.9k lines vs current `main`, +consistent with the prior pass's "too large" call) — left as-is otherwise. +The three findings named for re-verification this cycle +(`ProviderEmbeddingBatchBackend.submit` concurrency, `chat()` deadline +propagation, `zdr_only` leaking into provider payloads) were checked against +the PR's current head: the first two are already resolved there (Devin's +"Caller deadline is ignored on chat passthrough" thread is marked resolved, +and `submit`/`_run_job` already serialize every state transition under +`self._registry.lock(...)` with a bounded `ThreadPoolExecutor`), and +`zdr_only` does not exist anywhere in this PR's diff — that finding belongs to +**PR #911** instead (open, unresolved CodeRabbit thread on `server.py`'s +`_validate_zdr_only` not stripping the field from provider request bodies), +not #857; apparently conflated across PRs in an earlier pass's notes. Of +#857's 21 still-unresolved review threads, two were narrowly safe to fix +without touching the stale-merge problem, pushed as `9b9f9e4d` (a plain +commit on the existing head, no merge, no rebase): +- `CostRoutingCoordinator.__init__`'s readiness-recovery loop and + `_run_provider_readiness_job` both indexed `self._readiness_jobs[job_id]` + with no presence check; a durable (Valkey/Redis) backend can expire that + document's TTL between the key listing and the lookup, raising `KeyError` + out of `__init__` (failing server construction) or silently killing the + readiness worker thread (leaving the job stuck `queued`/`running` + forever). Both sites now check `isinstance(..., dict)` and return/continue. +- `tests/test_naruon_ecosystem_connector.py` called + `urllib.request.urlopen(req)` with no timeout, unlike every other HTTP test + in the file (`timeout=10`); added it. +Validated with the Rust `_token_packer` extension built locally (`maturin +develop --release`, needed because `build_token_counter` now hard-requires it +— itself one of the 21 still-open findings, left alone): focused suite 54 +passed; full suite `2748 passed, 1 skipped, 1 failed` (same `fast-mlsirm` +sandbox gap as above). The remaining ~19 unresolved threads (Dockerfile +`test-runner` stage missing the `orchestrator` user — Major; unbounded +OpenRouter endpoint enumeration; a resolver workflow pinned to a mutable ref; +several Minor/Info items) were left untouched — the Dockerfile one needs a +real `docker build` to fix safely (no daemon available in this sandbox), and +the rest touch enough surrounding logic to risk the kind of regression this +PR has already spent 268 commits chasing. + +**#906** (`feat/nim-benchmark-rebuild-20260828`) was reported `dirty` by +GitHub's cached `mergeable_state`; a real trial merge of `origin/main` showed +the branch was NOT irreconcilably diverged as `dirty` implied — the only +textual conflict, across all 28 changed files plus everything `main` gained +over the PR's stale base (33 commits), was in `CHANGELOG.md` (both sides +appended bullets to the same `### Added`/`### Fixed` region). Resolved by +keeping both sides' bullets under the file's one-header-per-type-per-version +convention and merging `origin/main` into the PR branch (a merge commit; no +rebase, no history rewritten). That merge then surfaced two real, narrow +regressions against this PR's own test suite, both fixed and pushed together +as `7ba5fefc`: +- `tests/test_nim_benchmark_workflow_contract.py` read + `.github/workflows/tests.yml`, which `main` renamed to `ci.yml` in + `9b0a356d` ("use conventional workflow filename") sometime in those 33 + commits; the `nim_benchmark_quality` job content the tests check for is + present and intact under the new name — repointed both reads. +- `tests/test_nim_benchmark_release_acceptance.py:: + test_budgeted_client_fallback_and_transport_errors` matched the old error + string `"provider .* request failed"`. `main`'s new + `contextual_orchestrator/provider_errors.py` (PR #879) reclassifies + provider HTTP failures through `ProviderUpstreamError` (still a + `RuntimeError` subclass) with the fixed message `"provider rejected the + request with HTTP {status}"` — updated the match regex. + +One more failure surfaced by the full suite, `tests/ +test_nim_benchmark_release_acceptance.py:: +test_smoke_manifest_cannot_authorize_production_routing`, is **not** caused +by this merge: it was verified to fail identically — same +`configured_total_token_budget=1280` vs `observed_budget_tokens=1283` on task +`trick_arithmetic_lily_pads`/policy `conduct_bounded` — on this PR's own +unmerged head `b0167b08`, before touching `main` at all. That contradicts the +PR description's claimed "NIM focused and release/workflow tests: 112 +passed." This pass left it untouched rather than loosening the equal-budget +assertion or the `30`/`0.9` evidence thresholds without the PR author's input +on why observed token usage grew by exactly 3 tokens for that one locked +task; it needs the author's judgment (a legitimate token-counting fix +elsewhere in the 32-commit branch history vs. an actual regression), not a +bot's guess. Full suite after both merge-fixes: `2797 passed, 2 failed` (the +token-budget gap above, plus the same sandbox-only `fast-mlsirm` gap). +`opencode-review` and `strix` were already failing on this PR before the +merge for the same org-wide systemic reason (the `strix` job's own log shows +it calling out to `api.opencode.ai`, consistent with `AGENTS.md`'s +"OpenCode/Noema/Strix share this repo's gateway backend" migration note); +`noema-review` was passing even pre-merge. None of this is a new regression +from the merge itself. + +Nothing was merged to protected `main` this cycle — the org-wide +`opencode-review`/`noema-review` gate blocks every open PR here until +`ContextualWisdomLab/.github#1422` lands; that PR remains blocked on its own +`pull_request_target` trust-boundary deadlock and is out of this repo's +control. No new PRs had opened since the prior pass. + ## 2026-08-29 batch-routing object-authorization slice Protected `main` remains diff --git a/examples/nim_pricing_scenario.json b/examples/nim_pricing_scenario.json new file mode 100644 index 000000000..b9b1b4a5b --- /dev/null +++ b/examples/nim_pricing_scenario.json @@ -0,0 +1,10 @@ +{ + "scenario_version": "2026-08-04.1", + "scenario_status": "example_unreviewed", + "scenario_notes": "Schema-demonstration scenario for dry runs and tests. These are HYPOTHETICAL USD-per-million-token assumptions, not authoritative NVIDIA rates: the hosted NIM catalog is currently free to the caller (actual cost 0). A live paid-cost analysis requires a reviewed scenario ('scenario_status': 'reviewed') supplied by the operator; models absent from this table are honestly reported as 'unknown'. Rates below deliberately price only some dry-run models so the 'unknown' path stays exercised.", + "usd_per_million_tokens": { + "dryrun/chat-basic": {"input": 0.2, "output": 0.6}, + "dryrun/chat-vision": {"input": 0.35, "output": 1.1}, + "dryrun/chat-omni": {"input": 0.5, "output": 1.6} + } +} diff --git a/examples/nim_task_manifest.json b/examples/nim_task_manifest.json new file mode 100644 index 000000000..047558229 --- /dev/null +++ b/examples/nim_task_manifest.json @@ -0,0 +1,233 @@ +{ + "manifest_version": "2026-08-07.1", + "manifest_notes": "Immutable task ids; expected answers and strict aliases are scoring-side only and are never injected into model prompts (no test-set leakage). The locked split contains thirty original, objectively scored tasks so the scheduled paired comparison can reach the repository's declared non-smoke evidence floor. Legacy scorer fields preserve authoring compatibility; the supported benchmark derives versioned complete-answer semantics, including explicit aliases and task-specific case sensitivity. The exploratory split remains tuning-only and never enters headline comparisons.", + "tasks": [ + { + "task_id": "trick_arithmetic_bat_ball", + "split": "locked", + "prompt": "A bat and a ball cost $1.10 in total. The bat costs $1.00 more than the ball. How much does the ball cost, in dollars? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "0.05"} + }, + { + "task_id": "constant_speed_distance", + "split": "locked", + "prompt": "A car travels 60 kilometers in 40 minutes. At that same speed, how many kilometers does it travel in 100 minutes? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "150"} + }, + { + "task_id": "trick_arithmetic_lily_pads", + "split": "locked", + "prompt": "A patch of lily pads doubles in size every day. It covers a whole lake in 48 days. After how many days did it cover half the lake? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "47"} + }, + { + "task_id": "letter_counting_strawberry", + "split": "locked", + "prompt": "How many times does the letter r appear in the word strawberry? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "3"} + }, + { + "task_id": "unit_conversion_km_miles", + "split": "locked", + "prompt": "A road is exactly 160.9344 kilometers long. How long is it in miles? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "100"} + }, + { + "task_id": "logic_trap_month_days", + "split": "locked", + "prompt": "Some months have 30 days and some have 31. How many months of the year have 28 days? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "12"} + }, + { + "task_id": "capital_recall_france", + "split": "locked", + "prompt": "Name the capital city of France. Answer with the city name only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "Paris"} + }, + { + "task_id": "capital_recall_australia", + "split": "locked", + "prompt": "Name the capital city of Australia. Answer with the city name only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "Canberra"} + }, + { + "task_id": "sequence_next_fibonacci", + "split": "locked", + "prompt": "What number comes next in this sequence: 1, 1, 2, 3, 5, 8, 13? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "21"} + }, + { + "task_id": "digit_sum_reasoning", + "split": "locked", + "prompt": "Multiply 12 by 12, then add 56 to the result. Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "200"} + }, + { + "task_id": "linear_equation_solution", + "split": "locked", + "prompt": "Solve 3x + 5 = 26 for x. Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "7"} + }, + { + "task_id": "combination_pair_count", + "split": "locked", + "prompt": "Five students each shake hands with every other student exactly once. How many handshakes occur? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "10"} + }, + { + "task_id": "fair_coin_probability", + "split": "locked", + "prompt": "A fair coin is flipped twice. What is the probability that both flips are heads? Answer with a decimal number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "0.25"} + }, + { + "task_id": "leap_year_day_count", + "split": "locked", + "prompt": "How many days are in the year 2024? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "366"} + }, + { + "task_id": "temperature_conversion_celsius", + "split": "locked", + "prompt": "Convert 68 degrees Fahrenheit to degrees Celsius. Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "20"} + }, + { + "task_id": "next_prime_number", + "split": "locked", + "prompt": "What is the smallest prime number greater than 29? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "31"} + }, + { + "task_id": "arithmetic_mean_value", + "split": "locked", + "prompt": "What is the arithmetic mean of 2, 4, and 9? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "5"} + }, + { + "task_id": "percentage_discount_price", + "split": "locked", + "prompt": "An item costs $80 before a 25 percent discount. What is the discounted price in dollars? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "60"} + }, + { + "task_id": "rectangle_perimeter_value", + "split": "locked", + "prompt": "A rectangle has side lengths 7 centimeters and 4 centimeters. What is its perimeter in centimeters? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "22"} + }, + { + "task_id": "constant_rate_travel", + "split": "locked", + "prompt": "A train travels at 72 kilometers per hour for 2.5 hours. How many kilometers does it travel? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "180"} + }, + { + "task_id": "binary_decimal_conversion", + "split": "locked", + "prompt": "Convert the binary numeral 101101 to base ten. Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "45"} + }, + { + "task_id": "roman_numeral_conversion", + "split": "locked", + "prompt": "Convert the Roman numeral XLII to an Arabic numeral. Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "42"} + }, + { + "task_id": "elapsed_clock_time", + "split": "locked", + "prompt": "A meeting starts at 14:35 and lasts 95 minutes. At what time does it end? Answer in HH:MM format only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "16:10"} + }, + { + "task_id": "syllogism_validity_classification", + "split": "locked", + "prompt": "All poets are readers. Some readers are cyclists. Therefore some poets are cyclists. Classify the conclusion in one word.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "invalid"} + }, + { + "task_id": "string_reversal_result", + "split": "locked", + "prompt": "Reverse the letters of the English word stressed. Answer with the resulting word only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "desserts"} + }, + { + "task_id": "korean_word_translation", + "split": "locked", + "prompt": "Translate the Korean fruit word 사과 into English. Answer with one word only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "apple"} + }, + { + "task_id": "chemical_symbol_gold", + "split": "locked", + "prompt": "What is the chemical symbol for gold? Answer with the symbol only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "Au", "strict_case_sensitive": true} + }, + { + "task_id": "largest_ocean_name", + "split": "locked", + "prompt": "Name the largest ocean on Earth. Answer with its name only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": { + "substring": "Pacific", + "strict_texts": ["Pacific", "Pacific Ocean"] + } + }, + { + "task_id": "capital_recall_canada", + "split": "locked", + "prompt": "Name the capital city of Canada. Answer with the city name only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "Ottawa"} + }, + { + "task_id": "square_root_integer", + "split": "locked", + "prompt": "What is the positive integer square root of 144? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "12"} + }, + { + "task_id": "exploratory_summary_probe", + "split": "exploratory", + "prompt": "Summarize in one word the mood of a calm sunny morning by the sea.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "peace"} + }, + { + "task_id": "exploratory_translation_probe", + "split": "exploratory", + "prompt": "Translate the English word hello into French. Answer with one word only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "bonjour"} + } + ] +} diff --git a/fuzz/corpus/nim_catalog/hostile_entries.json b/fuzz/corpus/nim_catalog/hostile_entries.json new file mode 100644 index 000000000..12e2f3904 --- /dev/null +++ b/fuzz/corpus/nim_catalog/hostile_entries.json @@ -0,0 +1 @@ +{"data": [{"id": "dup/model"}, {"id": "dup/model"}, {"owned_by": "no-id"}, "not-an-object", {"id": " "}, {"id": 42}, {"id": "ok/model", "owned_by": 99}]} diff --git a/fuzz/corpus/nim_catalog/valid_catalog.json b/fuzz/corpus/nim_catalog/valid_catalog.json new file mode 100644 index 000000000..0bffa69d7 --- /dev/null +++ b/fuzz/corpus/nim_catalog/valid_catalog.json @@ -0,0 +1 @@ +{"object": "list", "data": [{"id": "meta/llama-3.1-8b-instruct", "owned_by": "meta"}, {"id": "nvidia/nv-embed-v1", "owned_by": "nvidia"}]} diff --git a/fuzz/fuzz_nim_catalog.py b/fuzz/fuzz_nim_catalog.py new file mode 100644 index 000000000..1303044ca --- /dev/null +++ b/fuzz/fuzz_nim_catalog.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Atheris coverage-guided harness: NIM benchmark model-catalog parser. + +Surface: ``nim_benchmark.parse_model_catalog_body`` -- the untrusted-input +parser for the provider's ``GET /v1/models`` response body. + +Run locally (needs a permissive-licensed build of Atheris, Apache-2.0):: + + python fuzz/fuzz_nim_catalog.py -atomic_step -max_total_time=60 fuzz/corpus/nim_catalog +""" + +import sys +from pathlib import Path + +import atheris + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +with atheris.instrument_imports(): + from fuzz.targets import exercise_nim_catalog + + +def one_input(data: bytes) -> None: + """Feed one fuzzer-generated body to the catalog parser invariants.""" + exercise_nim_catalog(data) + + +def main() -> None: + """Set up and run the Atheris fuzzing loop.""" + atheris.Setup(sys.argv, one_input) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzz/targets.py b/fuzz/targets.py index 3cc3641e3..79b099508 100644 --- a/fuzz/targets.py +++ b/fuzz/targets.py @@ -422,3 +422,45 @@ def exercise_structured_output_error(content: str, schema: Any) -> None: } result = _structured_output_error(content, response_format) assert result in {None, "invalid_json", "schema_missing", "schema_violation"} + +def exercise_nim_catalog(raw: bytes) -> None: + """Drive the NIM benchmark model-catalog parser over arbitrary bytes. + + ``parse_model_catalog_body`` consumes an untrusted provider response + (``GET /v1/models``). Invariants for arbitrary input: structural failures + surface only as ``CatalogDiscoveryError`` (or a plain json RecursionError on + attacker-depth nesting); successful parses are deduplicated, sorted (immune + to provider response-order drift), machine-readably annotated, and stable + under reparse. + """ + from contextual_orchestrator.nim_benchmark import ( + CatalogDiscoveryError, + parse_model_catalog_body, + ) + + try: + catalog = parse_model_catalog_body(raw) + except CatalogDiscoveryError: + return + except RecursionError: + # json depth blowups mirror the request-body parser's accepted failure. + return + + assert set(catalog) == {"models", "duplicate_model_ids", "invalid_entries"} + model_ids = [row["model_id"] for row in catalog["models"]] + assert model_ids == sorted(model_ids), "catalog must be order-drift immune" + assert len(model_ids) == len(set(model_ids)), "catalog must be deduplicated" + for row in catalog["models"]: + assert isinstance(row["model_id"], str) and row["model_id"].strip() + assert isinstance(row["owned_by"], str) + for entry in catalog["invalid_entries"]: + assert entry["invalid_reason"] in {"entry_not_an_object", "missing_model_id"} + assert catalog["duplicate_model_ids"] == sorted(catalog["duplicate_model_ids"]) + + # The whole result must be JSON-serialisable, and reparsing the surviving + # models must be a fixed point (parse . serialize . parse == parse). + reserialized = json.dumps( + {"data": [{"id": row["model_id"], "owned_by": row["owned_by"]} for row in catalog["models"]]} + ).encode("utf-8") + if catalog["models"]: + assert parse_model_catalog_body(reserialized)["models"] == catalog["models"] diff --git a/requirements-opencode-review-ci.in b/requirements-opencode-review-ci.in index e1035468c..f6c44e8d9 100644 --- a/requirements-opencode-review-ci.in +++ b/requirements-opencode-review-ci.in @@ -1,4 +1,5 @@ uv==0.12.5 +setuptools==84.0.0 pytest==9.0.3 pytest-cov>=5 coverage>=7.15.4 diff --git a/requirements-opencode-review-ci.txt b/requirements-opencode-review-ci.txt index 451d84aeb..05606bec9 100644 --- a/requirements-opencode-review-ci.txt +++ b/requirements-opencode-review-ci.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --generate-hashes --output-file=requirements-opencode-review-ci.txt requirements-opencode-review-ci.in -# +# This file was autogenerated by uv via the following command: +# uv pip compile requirements-opencode-review-ci.in --generate-hashes --output-file requirements-opencode-review-ci.txt attrs==26.1.0 \ --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 @@ -16,7 +12,7 @@ colorama==0.4.6 \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 # via interrogate -coverage[toml]==7.15.4 \ +coverage==7.15.4 \ --hash=sha256:002a438859f7b430bc99afeaf01a6d187dad1d0dc907b64cdeffc632a5db8fd8 \ --hash=sha256:050467a7983b8e2fe7dd41a78bb30c3e7f8c0b8cafda14b1c46f8b5e3cf2dd3c \ --hash=sha256:050f66a08805acb5b8a23c6d4a517b1ecf82c08e81ed0e4bd727df065e5c6624 \ @@ -141,10 +137,6 @@ coverage[toml]==7.15.4 \ # via # -r requirements-opencode-review-ci.in # pytest-cov -exceptiongroup==1.3.1 \ - --hash=sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219 \ - --hash=sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598 - # via pytest iniconfig==2.3.0 \ --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 @@ -181,66 +173,14 @@ pytest-cov==7.1.0 \ --hash=sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2 \ --hash=sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678 # via -r requirements-opencode-review-ci.in +setuptools==84.0.0 \ + --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ + --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 + # via -r requirements-opencode-review-ci.in tabulate==0.10.0 \ --hash=sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d \ --hash=sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3 # via interrogate -tomli==2.4.1 \ - --hash=sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853 \ - --hash=sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe \ - --hash=sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5 \ - --hash=sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d \ - --hash=sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd \ - --hash=sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26 \ - --hash=sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54 \ - --hash=sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6 \ - --hash=sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c \ - --hash=sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a \ - --hash=sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd \ - --hash=sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f \ - --hash=sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5 \ - --hash=sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9 \ - --hash=sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662 \ - --hash=sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9 \ - --hash=sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1 \ - --hash=sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585 \ - --hash=sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e \ - --hash=sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c \ - --hash=sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41 \ - --hash=sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f \ - --hash=sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085 \ - --hash=sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15 \ - --hash=sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7 \ - --hash=sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c \ - --hash=sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36 \ - --hash=sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076 \ - --hash=sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac \ - --hash=sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8 \ - --hash=sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232 \ - --hash=sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece \ - --hash=sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a \ - --hash=sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897 \ - --hash=sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d \ - --hash=sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4 \ - --hash=sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917 \ - --hash=sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396 \ - --hash=sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a \ - --hash=sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc \ - --hash=sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba \ - --hash=sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f \ - --hash=sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257 \ - --hash=sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30 \ - --hash=sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf \ - --hash=sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9 \ - --hash=sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049 - # via - # coverage - # interrogate - # pytest -typing-extensions==4.16.0 \ - --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ - --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 - # via exceptiongroup uv==0.12.5 \ --hash=sha256:1a06c8bc4d43b5f6c1e3f2ae3d0f6455b07515f762516f95e52e6c0cbccedf15 \ --hash=sha256:2bd62134e56af35b9cf017aaf8ae41a605d6501dd49afc35b70b544a45dd8354 \ diff --git a/tests/fuzz/test_fuzz_properties.py b/tests/fuzz/test_fuzz_properties.py index e58858b58..d5cf9688a 100644 --- a/tests/fuzz/test_fuzz_properties.py +++ b/tests/fuzz/test_fuzz_properties.py @@ -20,6 +20,7 @@ exercise_agent_config, exercise_model_judge_reply, exercise_models_dev_cost, + exercise_nim_catalog, exercise_orchestration, exercise_pii_key, exercise_provider_model_payload, @@ -150,3 +151,35 @@ def test_structured_output_validation_never_crashes(content: str, schema: object @given(_json_values) def test_reasoning_effort_profile_never_crashes(value: object) -> None: exercise_reasoning_effort_profile(value) + +@_SETTINGS +@given(st.binary(max_size=4096)) +def test_nim_catalog_never_crashes_on_raw_bytes(raw: bytes) -> None: + exercise_nim_catalog(raw) + + +# Catalog-shaped adversarial entries: wrong types, missing ids, duplicates. +_catalog_entry = ( + st.none() + | st.text(max_size=16) + | st.integers() + | st.fixed_dictionaries( + {}, + optional={ + "id": st.text(max_size=20) | st.integers() | st.none() | st.just("dup/model"), + "owned_by": st.text(max_size=12) | st.integers() | st.none(), + }, + ) +) + + +@_SETTINGS +@given(st.lists(_catalog_entry, max_size=8).map(lambda entries: json.dumps({"data": entries}).encode("utf-8"))) +def test_nim_catalog_on_structured_entries(raw: bytes) -> None: + exercise_nim_catalog(raw) + + +@_SETTINGS +@given(_json_values.map(lambda v: json.dumps(v).encode("utf-8"))) +def test_nim_catalog_on_arbitrary_json(raw: bytes) -> None: + exercise_nim_catalog(raw) diff --git a/tests/test_cli_auth.py b/tests/test_cli_auth.py index 831097eb3..e2a6d09c2 100644 --- a/tests/test_cli_auth.py +++ b/tests/test_cli_auth.py @@ -237,6 +237,20 @@ def test_main_accepts_explicit_argv_without_mutating_process_arguments() -> None assert security.auth_token == expected_value +def test_main_dispatches_nim_benchmark_from_explicit_argv() -> None: + with patch( + "contextual_orchestrator.nim_benchmark.run_benchmark_cli", + return_value=7, + ) as run_benchmark_cli: + try: + main(["nim-benchmark", "--dry-run"]) + except SystemExit as exc: + assert exc.code == 7 + else: # pragma: no cover + raise AssertionError("benchmark CLI must return its exit code") + run_benchmark_cli.assert_called_once_with(["--dry-run"]) + + def test_invalid_local_provider_options_fail_at_parser_boundary() -> None: invalid_options = ( (["--local-concurrency", "0"], "positive integer"), diff --git a/tests/test_nim_benchmark.py b/tests/test_nim_benchmark.py new file mode 100644 index 000000000..43442d20a --- /dev/null +++ b/tests/test_nim_benchmark.py @@ -0,0 +1,2214 @@ +"""NIM benchmark harness contracts — discovery, all-modality probes, fair eval. + +Everything here runs fully offline: provider behavior is injected through the +transport seam, evaluation workers ride the mock:// path, and the credential +registry is a fresh in-memory KV per test. Adversarial coverage follows the +issue contract: malformed catalogs, duplicate ids, unsupported capabilities, +partial results, non-finite token/cost values, rate limits, timeouts, +response-order drift, and secret redaction. +""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import socket +import tempfile +import threading +import urllib.error +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import nim_benchmark as nb # noqa: E402 +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + NotConfigured, + register_credential, + set_backend, +) +from contextual_orchestrator.orchestrator import ( # noqa: E402 + ModelAgent, + ModelClient, + TaskOrchestrator, + _FastMLSIJudgeAdapter, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +TASK_MANIFEST_PATH = str(REPO_ROOT / "examples" / "nim_task_manifest.json") +PRICING_SCENARIO_PATH = str(REPO_ROOT / "examples" / "nim_pricing_scenario.json") +FAKE_ENDPOINT = "https://nim.example.test/v1" + + +@pytest.fixture(autouse=True) +def _fresh_backend(): + """Isolated in-memory KV and a clean benchmark env var for every test.""" + set_backend(InMemoryCredentialBackend()) + saved_env = os.environ.pop(nb.NIM_CREDENTIAL_NAME, None) + try: + yield + finally: + set_backend(None) + if saved_env is not None: + os.environ[nb.NIM_CREDENTIAL_NAME] = saved_env + + +def _ok_json(payload: object) -> tuple[int, bytes]: + return 200, json.dumps(payload).encode("utf-8") + + +def _fixed_transport(status: int, body: bytes): + def transport(method, url, headers, body_bytes): + return status, body + + return transport + + +def _mini_manifest(task_count: int = 2) -> dict: + tasks = [ + { + "task_id": f"locked_task_{index}", + "split": "locked", + "prompt": f"Question number {index}?", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "zebra"}, + } + for index in range(task_count) + ] + return {"manifest_version": "test.1", "tasks": tasks} + + +def _mock_agents(*model_ids: str) -> list[ModelAgent]: + taken: set[str] = set() + return [ + ModelAgent( + id=nb.sanitize_worker_agent_id(model_id, taken), + model=model_id, + base_url="mock://nim-test", + credential_key=nb.NIM_CREDENTIAL_NAME, + tags=("reasoning", "writing"), + ) + for model_id in model_ids + ] + + +# -------------------------------------------------------------------------- +# Egress guard + default transport +# -------------------------------------------------------------------------- + + +def test_endpoint_guard_rejects_http() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.require_public_https_endpoint("http://nim.example.test/v1") + + +def test_endpoint_guard_rejects_missing_host() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.require_public_https_endpoint("https:///v1") + + +def _patched_getaddrinfo(ip_address: str): + return lambda *args, **kwargs: [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip_address, 443)) + ] + + +def test_endpoint_guard_rejects_private_address() -> None: + original = socket.getaddrinfo + socket.getaddrinfo = _patched_getaddrinfo("10.0.0.8") + try: + with pytest.raises(nb.BenchmarkContractError): + nb.require_public_https_endpoint(FAKE_ENDPOINT) + finally: + socket.getaddrinfo = original + + +def test_endpoint_guard_accepts_public_address() -> None: + original = socket.getaddrinfo + socket.getaddrinfo = _patched_getaddrinfo("93.184.216.34") + try: + nb.require_public_https_endpoint(FAKE_ENDPOINT) + finally: + socket.getaddrinfo = original + + +class _FakeDirectResponse: + """Minimal response returned by the pinned HTTPS connection test seam.""" + + def __init__(self, status: int, body: bytes) -> None: + self.status = status + self._body = body + self.closed = False + + def read(self, maximum_bytes: int = -1) -> bytes: + if maximum_bytes < 0: + return self._body + return self._body[:maximum_bytes] + + def close(self) -> None: + self.closed = True + + +class _FakeDirectConnection: + """Scripted pinned connection that records address and authority evidence.""" + + plans: list[object] = [] + instances: list["_FakeDirectConnection"] = [] + + def __init__(self, server_hostname, pinned_ip, port, timeout, context) -> None: + self.server_hostname = server_hostname + self.pinned_ip = pinned_ip + self.port = port + self.timeout = timeout + self.context = context + self.method = "" + self.target = "" + self.body = None + self.headers = {} + self.closed = False + self._plan = type(self).plans.pop(0) + type(self).instances.append(self) + + def request(self, method, target, body, headers) -> None: + self.method = method + self.target = target + self.body = body + self.headers = headers + if isinstance(self._plan, BaseException): + raise self._plan + + def getresponse(self): + return self._plan + + def close(self) -> None: + self.closed = True + + +def _install_direct_transport_fakes(monkeypatch, plans, addresses=("93.184.216.34",)): + """Install deterministic DNS and connection seams for one transport test.""" + resolution_calls = [] + + def resolve(host, port, label): + resolution_calls.append((host, port, label)) + return addresses + + _FakeDirectConnection.plans = list(plans) + _FakeDirectConnection.instances = [] + monkeypatch.setattr(nb, "validated_public_addresses", resolve) + monkeypatch.setattr(nb, "PinnedHTTPSConnection", _FakeDirectConnection) + return resolution_calls + + +def test_default_transport_returns_status_and_revalidates_each_request( + monkeypatch, +) -> None: + first_response = _FakeDirectResponse(200, b"body-one") + second_response = _FakeDirectResponse(200, b"body-two") + resolution_calls = _install_direct_transport_fakes( + monkeypatch, + [first_response, second_response], + ) + + transport = nb.build_default_transport(timeout_seconds=5.0) + first = transport("GET", f"{FAKE_ENDPOINT}/models", {}, None) + second = transport("GET", f"{FAKE_ENDPOINT}/models;format=json?second=1", {}, None) + + assert first == (200, b"body-one") + assert second == (200, b"body-two") + assert resolution_calls == [ + ("nim.example.test", 443, "NIM benchmark"), + ("nim.example.test", 443, "NIM benchmark"), + ] + assert all(response.closed for response in (first_response, second_response)) + assert all(connection.closed for connection in _FakeDirectConnection.instances) + assert _FakeDirectConnection.instances[0].server_hostname == "nim.example.test" + assert _FakeDirectConnection.instances[0].pinned_ip == "93.184.216.34" + assert ( + _FakeDirectConnection.instances[1].target == "/v1/models;format=json?second=1" + ) + + +def test_default_transport_returns_http_error_status_with_body(monkeypatch) -> None: + response = _FakeDirectResponse(429, b"slow down") + _install_direct_transport_fakes(monkeypatch, [response]) + assert nb.build_default_transport(5.0)( + "POST", f"{FAKE_ENDPOINT}/chat/completions", {}, b"{}" + ) == (429, b"slow down") + assert response.closed is True + + +def test_default_transport_rejects_oversized_response_and_closes_resources( + monkeypatch, +) -> None: + """A provider cannot exhaust memory with an unbounded response body.""" + response = _FakeDirectResponse(200, b"x" * (nb.MAX_PROVIDER_RESPONSE_BYTES + 1)) + _install_direct_transport_fakes(monkeypatch, [response]) + + with pytest.raises(nb.BenchmarkContractError, match="response exceeds"): + nb.build_default_transport(5.0)("GET", f"{FAKE_ENDPOINT}/models", {}, None) + + assert response.closed is True + assert _FakeDirectConnection.instances[0].closed is True + + +def test_default_transport_rejects_redirect_without_following(monkeypatch) -> None: + response = _FakeDirectResponse(302, b"redirect") + _install_direct_transport_fakes(monkeypatch, [response]) + with pytest.raises(nb.BenchmarkContractError, match="redirects are not permitted"): + nb.build_default_transport(5.0)( + "POST", f"{FAKE_ENDPOINT}/chat/completions", {}, b"{}" + ) + assert response.closed is True + + +def test_default_transport_falls_back_only_to_another_validated_address( + monkeypatch, +) -> None: + response = _FakeDirectResponse(200, b"catalog") + _install_direct_transport_fakes( + monkeypatch, + [OSError("first pin failed"), response], + addresses=("93.184.216.34", "93.184.216.35"), + ) + result = nb.build_default_transport(5.0)("GET", f"{FAKE_ENDPOINT}/models", {}, None) + assert result == (200, b"catalog") + assert [item.pinned_ip for item in _FakeDirectConnection.instances] == [ + "93.184.216.34", + "93.184.216.35", + ] + + +def test_default_transport_reports_failure_after_every_pin_fails(monkeypatch) -> None: + _install_direct_transport_fakes( + monkeypatch, + [OSError("first pin failed"), OSError("second pin failed")], + addresses=("93.184.216.34", "93.184.216.35"), + ) + with pytest.raises(urllib.error.URLError, match="second pin failed"): + nb.build_default_transport(5.0)("GET", f"{FAKE_ENDPOINT}/models", {}, None) + + +# -------------------------------------------------------------------------- +# Request budget +# -------------------------------------------------------------------------- + + +def test_request_budget_rejects_non_positive_cap() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.RequestBudget(0) + + +def test_request_budget_spends_then_exhausts() -> None: + budget = nb.RequestBudget(2) + assert budget.try_spend() and budget.try_spend() + assert not budget.try_spend() + assert budget.requests_spent == 2 + with pytest.raises(nb.BenchmarkBudgetError): + budget.spend_or_fail() + + +def test_budgeted_client_charges_each_chat_call() -> None: + budget = nb.RequestBudget(1) + client = nb._BudgetedModelClient(budget) + agent = _mock_agents("dryrun/chat-basic")[0] + assert client.max_retries == 0 + assert client.chat(agent, [{"role": "user", "content": "hello there"}]) + with pytest.raises(nb.BenchmarkBudgetError): + client.chat(agent, [{"role": "user", "content": "over budget"}]) + + +def test_budgeted_evaluation_transport_failures_are_fail_closed() -> None: + agent = ModelAgent( + "worker_one", + "vendor/model-a", + FAKE_ENDPOINT, + provider_name="nvidia_nim", + ) + + def client_for(result): + def transport(method, url, headers, body): + if isinstance(result, Exception): + raise result + return result + + return nb._BudgetedModelClient(nb.RequestBudget(2), transport=transport) + + for result, pattern in ( + (nb.BenchmarkContractError("redirect"), "redirect"), + ((401, b"{}"), "credential"), + ((200, b"not-json"), "valid JSON"), + ((200, b"[]"), "must be an object"), + ): + client = client_for(result) + with pytest.raises((nb.BenchmarkContractError, nb.BenchmarkAuthError), match=pattern): + client._send(agent, {}) + if isinstance(result, nb.BenchmarkContractError) or result[0] == 200: + assert client.benchmark_contract_error is not None + + for result, expected_error in ( + (nb.BenchmarkContractError("oversized"), nb.BenchmarkContractError), + ((403, b"{}"), nb.BenchmarkAuthError), + ((500, b"{}"), urllib.error.HTTPError), + ((200, b"not-json"), nb.BenchmarkContractError), + ((200, b"[]"), nb.BenchmarkContractError), + ): + client = client_for(result) + with pytest.raises(expected_error): + client.proxy_send_once(agent, "responses", {}) + + +def test_structured_judge_uses_transport_and_both_request_limits() -> None: + calls = [] + + def transport(method, url, headers, body): + calls.append((method, url, body)) + return 200, json.dumps( + { + "choices": [{"message": {"content": '{"score": 1}'}}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2}, + } + ).encode() + + budget = nb.RequestBudget(1) + delegate = nb._BudgetedModelClient(budget, transport=transport) + cell = nb.EqualBudgetModelClient(delegate, total_token_budget=100, maximum_calls=1) + agent = ModelAgent( + "judge_agent", + "vendor/judge-model", + FAKE_ENDPOINT, + provider_name="nvidia_nim", + tags=("verifier",), + ) + adapter = _FastMLSIJudgeAdapter( + TaskOrchestrator([agent], client=cell), "answer", agent.id + ) + result = adapter.complete_structured( + [{"role": "user", "content": "judge"}], + response_format={"type": "json_object"}, + ) + assert result["trace"][0]["usage"] == { + "prompt_tokens": 3, + "completion_tokens": 2, + } + assert calls and calls[0][0:2] == ( + "POST", + f"{FAKE_ENDPOINT}/chat/completions", + ) + assert budget.requests_spent == 1 + assert cell.observed_calls == 1 + assert cell.observed_tokens == 5 + with pytest.raises(nb.PolicyTokenBudgetExceeded, match="maximum-call"): + adapter.complete_structured( + [{"role": "user", "content": "again"}], + response_format={"type": "json_object"}, + ) + assert budget.requests_spent == 1 + + +def test_equal_budget_structured_transport_alias_and_invalid_usage() -> None: + class StructuredDelegate(ModelClient): + def proxy_send(self, agent, endpoint, payload): + return { + "choices": [{"message": {"content": "ok"}}], + "usage": {"prompt_tokens": "unknown", "completion_tokens": None}, + } + + cell = nb.EqualBudgetModelClient( + StructuredDelegate(), total_token_budget=100, maximum_calls=1 + ) + response = cell.proxy_send_once( + _mock_agents("dryrun/chat-basic")[0], "responses", {"input": "judge"} + ) + assert response["usage"]["prompt_tokens"] == "unknown" + + +def test_equal_budget_client_forwards_delegate_controls() -> None: + delegate = ModelClient(max_output_tokens=32) + client = nb.EqualBudgetModelClient( + delegate, total_token_budget=128, maximum_calls=2 + ) + assert client.temperature == delegate.temperature + assert client.timeout == delegate.timeout + assert client.request_settings_snapshot() == delegate.request_settings_snapshot() + with client.request_settings(max_output_tokens=16): + assert client.request_settings_snapshot()["max_output_tokens"] == 16 + + +def test_equal_budget_client_fails_closed_on_call_and_prompt_limits() -> None: + client = nb.EqualBudgetModelClient( + ModelClient(), total_token_budget=20, maximum_calls=1 + ) + client.chat( + _mock_agents("dryrun/chat-basic")[0], [{"role": "user", "content": "hi"}] + ) + with pytest.raises(nb.PolicyTokenBudgetExceeded, match="maximum-call"): + client.chat( + _mock_agents("dryrun/chat-basic")[0], [{"role": "user", "content": "again"}] + ) + + tight = nb.EqualBudgetModelClient( + ModelClient(), total_token_budget=1, maximum_calls=1 + ) + with pytest.raises(nb.PolicyTokenBudgetExceeded, match="total-token"): + tight.chat( + _mock_agents("dryrun/chat-basic")[0], + [{"role": "user", "content": "x" * 100}], + ) + + +# -------------------------------------------------------------------------- +# Catalog parsing (adversarial) +# -------------------------------------------------------------------------- + + +def test_catalog_parse_rejects_invalid_utf8() -> None: + with pytest.raises(nb.CatalogDiscoveryError): + nb.parse_model_catalog_body(b"\xff\xfe\xfa") + + +def test_catalog_parse_rejects_invalid_json() -> None: + with pytest.raises(nb.CatalogDiscoveryError): + nb.parse_model_catalog_body(b"{not json") + + +def test_catalog_parse_rejects_non_object_and_missing_data() -> None: + with pytest.raises(nb.CatalogDiscoveryError): + nb.parse_model_catalog_body(b"[1, 2, 3]") + with pytest.raises(nb.CatalogDiscoveryError): + nb.parse_model_catalog_body(json.dumps({"data": "nope"}).encode("utf-8")) + + +def test_catalog_parse_records_invalid_and_duplicate_entries() -> None: + body = json.dumps( + { + "data": [ + {"id": "vendor/model-b", "owned_by": "vendor"}, + {"id": "vendor/model-b", "owned_by": "vendor"}, + "not-an-object", + {"owned_by": "vendor"}, + {"id": " ", "owned_by": "vendor"}, + {"id": 42}, + {"id": "vendor/model-a", "owned_by": 99}, + {"id": "vendor/model-b", "owned_by": "vendor"}, + ] + } + ).encode("utf-8") + catalog = nb.parse_model_catalog_body(body) + # Sorted output guards against provider response-order drift. + assert [row["model_id"] for row in catalog["models"]] == [ + "vendor/model-a", + "vendor/model-b", + ] + assert catalog["models"][0]["owned_by"] == "" # non-string owner coerced + assert catalog["duplicate_model_ids"] == ["vendor/model-b"] + reasons = {entry["invalid_reason"] for entry in catalog["invalid_entries"]} + assert reasons == {"entry_not_an_object", "missing_model_id"} + assert len(catalog["invalid_entries"]) == 4 + + +def test_catalog_order_drift_never_reorders_models() -> None: + forward = json.dumps( + {"data": [{"id": "a/model-one"}, {"id": "b/model-two"}]} + ).encode("utf-8") + reverse = json.dumps( + {"data": [{"id": "b/model-two"}, {"id": "a/model-one"}]} + ).encode("utf-8") + assert ( + nb.parse_model_catalog_body(forward)["models"] + == nb.parse_model_catalog_body(reverse)["models"] + ) + + +def test_discover_catalog_success_and_budget_charge() -> None: + budget = nb.RequestBudget(3) + catalog = nb.discover_model_catalog( + _fixed_transport(*_ok_json({"data": [{"id": "vendor/model-a"}]})), + FAKE_ENDPOINT, + "key", + budget, + ) + assert catalog["models"][0]["model_id"] == "vendor/model-a" + assert budget.requests_spent == 1 + + +def test_discover_catalog_fails_closed_on_auth_rejection() -> None: + for status in (401, 403): + with pytest.raises(nb.BenchmarkAuthError): + nb.discover_model_catalog( + _fixed_transport(status, b"{}"), + FAKE_ENDPOINT, + "key", + nb.RequestBudget(3), + ) + + +def test_discover_catalog_fails_closed_on_http_error() -> None: + with pytest.raises(nb.CatalogDiscoveryError): + nb.discover_model_catalog( + _fixed_transport(500, b"{}"), FAKE_ENDPOINT, "key", nb.RequestBudget(3) + ) + + +def test_discover_catalog_fails_closed_on_network_error() -> None: + def transport(method, url, headers, body): + raise urllib.error.URLError("dns failure") + + with pytest.raises(nb.CatalogDiscoveryError): + nb.discover_model_catalog(transport, FAKE_ENDPOINT, "key", nb.RequestBudget(3)) + + +def test_discover_catalog_fails_closed_on_dns_error() -> None: + def transport(method, url, headers, body): + raise socket.gaierror(-2, "name or service not known") + + with pytest.raises(nb.CatalogDiscoveryError): + nb.discover_model_catalog(transport, FAKE_ENDPOINT, "key", nb.RequestBudget(3)) + + +def test_discover_catalog_fails_closed_on_empty_inventory() -> None: + with pytest.raises(nb.CatalogDiscoveryError): + nb.discover_model_catalog( + _fixed_transport(*_ok_json({"data": []})), + FAKE_ENDPOINT, + "key", + nb.RequestBudget(3), + ) + + +# -------------------------------------------------------------------------- +# Capability probes — every NIM contract +# -------------------------------------------------------------------------- + + +def test_probe_registry_covers_every_nim_contract() -> None: + assert set(nb.CAPABILITY_PROBE_ORDER) == { + "chat_completion", + "text_completion", + "response_generation", + "text_embedding", + "image_understanding", + "video_understanding", + "audio_understanding", + "audio_transcription", + "audio_speech", + } + + +def test_probe_assets_are_deterministic_and_well_formed() -> None: + assert nb._tiny_wav_bytes() == nb._tiny_wav_bytes() + assert nb._tiny_wav_bytes().startswith(b"RIFF") + assert nb._image_data_uri().startswith("data:image/png;base64,") + assert nb._video_data_uri().startswith("data:video/mp4;base64,") + multipart = nb._multipart_transcription_body("vendor/asr-model") + assert b'name="model"' in multipart and b"vendor/asr-model" in multipart + assert b'filename="probe.wav"' in multipart and b"RIFF" in multipart + + +def test_response_validators_accept_and_reject_shapes() -> None: + assert nb._has_choice({"choices": [{"message": {"content": "x"}}]}) + assert not nb._has_choice({"choices": []}) and not nb._has_choice({}) + assert nb._has_embedding({"data": [{"embedding": [0.1]}]}) + assert not nb._has_embedding({"data": []}) + assert not nb._has_embedding({"data": ["oops"]}) + assert not nb._has_embedding({"data": [{"embedding": "oops"}]}) + assert not nb._has_embedding({}) + assert nb._has_response_output({"output_text": "x"}) + assert not nb._has_response_output({"unrelated": 1}) + assert nb._has_transcription_text({"text": "ok"}) + assert not nb._has_transcription_text({"text": 5}) + + +def test_probe_status_classification_table() -> None: + assert nb.classify_probe_status(200) == "supported" + for status in (400, 404, 405, 415, 422, 501): + assert nb.classify_probe_status(status) == "unsupported" + for status in (401, 403): + assert nb.classify_probe_status(status) == "auth_rejected" + assert nb.classify_probe_status(408) == "timeout" + assert nb.classify_probe_status(429) == "rate_limited" + assert nb.classify_probe_status(500) == "unavailable" + assert nb.classify_probe_status(302) == "failed" + + +def _probe(transport, capability_name="chat_completion"): + return nb.execute_capability_probe( + transport, FAKE_ENDPOINT, "key", "vendor/model-a", capability_name + ) + + +def test_probe_supported_chat() -> None: + row = _probe( + _fixed_transport(*_ok_json({"choices": [{"message": {"content": "OK"}}]})) + ) + assert row["probe_outcome"] == "supported" + assert row["http_status"] == 200 + + +def test_probe_timeout_and_network_failures() -> None: + def timeout_transport(method, url, headers, body): + raise socket.timeout("slow") + + def broken_transport(method, url, headers, body): + raise ConnectionResetError("reset") + + assert _probe(timeout_transport)["probe_outcome"] == "timeout" + row = _probe(broken_transport) + assert row["probe_outcome"] == "failed" + assert row["outcome_reason"].startswith("network_error:") + + +@pytest.mark.parametrize("status", [401, 403]) +def test_probe_auth_rejection_fails_closed(status: int) -> None: + with pytest.raises(nb.BenchmarkAuthError): + _probe(_fixed_transport(status, b"{}")) + + +def test_probe_unsupported_and_rate_limited() -> None: + assert _probe(_fixed_transport(404, b"{}"))["probe_outcome"] == "unsupported" + assert _probe(_fixed_transport(429, b"{}"))["probe_outcome"] == "rate_limited" + + +def test_probe_malformed_success_bodies() -> None: + assert ( + _probe(_fixed_transport(200, b"not json"))["probe_outcome"] + == "malformed_response" + ) + assert _probe(_fixed_transport(200, b"[]"))["probe_outcome"] == "malformed_response" + assert ( + _probe(_fixed_transport(*_ok_json({"choices": []})))["probe_outcome"] + == "malformed_response" + ) + assert ( + _probe(_fixed_transport(*_ok_json({"choices": [{}]})))["probe_outcome"] + == "malformed_response" + ) + + +def test_probe_binary_speech_contract() -> None: + supported = _probe( + _fixed_transport(200, b"RIFF\x00\x00\x00\x00WAVEaudio"), "audio_speech" + ) + assert supported["probe_outcome"] == "supported" + empty = _probe(_fixed_transport(200, b""), "audio_speech") + assert empty["probe_outcome"] == "malformed_response" + assert empty["outcome_reason"] == "http_200_without_audio_signature" + assert ( + _probe(_fixed_transport(200, b'{"error":"not audio"}'), "audio_speech")[ + "probe_outcome" + ] + == "malformed_response" + ) + + +def _rows(**outcome_by_capability: str) -> list[dict]: + return [ + {"capability_name": name, "probe_outcome": outcome} + for name, outcome in outcome_by_capability.items() + ] + + +def test_classification_covers_every_modality_class() -> None: + assert ( + nb.classify_model_capabilities( + _rows( + chat_completion="supported", + image_understanding="supported", + audio_understanding="supported", + ) + )["model_classification"] + == "omni_capable" + ) + assert ( + nb.classify_model_capabilities( + _rows(chat_completion="supported", image_understanding="supported") + )["model_classification"] + == "vision_chat_capable" + ) + assert ( + nb.classify_model_capabilities( + _rows(chat_completion="supported", video_understanding="supported") + )["model_classification"] + == "vision_chat_capable" + ) + assert ( + nb.classify_model_capabilities(_rows(chat_completion="supported"))[ + "model_classification" + ] + == "chat_capable" + ) + assert ( + nb.classify_model_capabilities(_rows(text_embedding="supported"))[ + "model_classification" + ] + == "embedding_only" + ) + assert ( + nb.classify_model_capabilities(_rows(text_completion="supported"))[ + "model_classification" + ] + == "completion_only" + ) + assert ( + nb.classify_model_capabilities(_rows(response_generation="supported"))[ + "model_classification" + ] + == "responses_only" + ) + assert ( + nb.classify_model_capabilities(_rows(audio_transcription="supported"))[ + "model_classification" + ] + == "audio_only" + ) + assert ( + nb.classify_model_capabilities(_rows(audio_speech="supported"))[ + "model_classification" + ] + == "audio_only" + ) + assert ( + nb.classify_model_capabilities(_rows(chat_completion="skipped"))[ + "model_classification" + ] + == "skipped" + ) + assert ( + nb.classify_model_capabilities( + _rows(chat_completion="rate_limited", text_embedding="unsupported") + )["model_classification"] + == "rate_limited" + ) + assert ( + nb.classify_model_capabilities( + _rows(chat_completion="unavailable", text_embedding="unsupported") + )["model_classification"] + == "unavailable" + ) + assert ( + nb.classify_model_capabilities( + _rows(chat_completion="timeout", text_embedding="unsupported") + )["model_classification"] + == "failed" + ) + assert ( + nb.classify_model_capabilities( + _rows(chat_completion="unsupported", text_embedding="unsupported") + )["model_classification"] + == "unsupported_for_contract" + ) + + +def test_classification_reports_chat_eligibility() -> None: + assert nb.classify_model_capabilities(_rows(chat_completion="supported"))[ + "chat_eligible" + ] + assert not nb.classify_model_capabilities(_rows(text_embedding="supported"))[ + "chat_eligible" + ] + + +def test_probe_models_rejects_bad_concurrency() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.probe_discovered_models( + [], + _fixed_transport(200, b"{}"), + FAKE_ENDPOINT, + "key", + nb.RequestBudget(1), + 0, + lambda: 0.0, + ) + + +def test_probe_models_sorted_despite_input_order_drift() -> None: + # Models arrive in reverse order; the snapshot must still come out sorted. + models = [ + {"model_id": "b/model-two", "owned_by": ""}, + {"model_id": "a/model-one", "owned_by": ""}, + ] + results = nb.probe_discovered_models( + models, + _fixed_transport(*_ok_json({"choices": [{"message": {"content": "OK"}}]})), + FAKE_ENDPOINT, + "key", + nb.RequestBudget(100), + 2, + lambda: 1234.0, + lambda: 0.0, + ) + assert [row["model_id"] for row in results] == ["a/model-one", "b/model-two"] + # The fixed transport answers every probe, so the model reads as omni. + assert results[0]["model_classification"] == "omni_capable" + assert results[0]["discovered_at_unix"] == 1234.0 + assert results[0]["endpoint"] == FAKE_ENDPOINT + + +def test_probe_models_share_one_run_timestamp() -> None: + timestamps = iter((1234.0, 9999.0, 9999.0)) + results = nb.probe_discovered_models( + [{"model_id": "a/model-one", "owned_by": ""}, {"model_id": "b/model-two", "owned_by": ""}], + _fixed_transport(*_ok_json({"choices": [{"message": {"content": "OK"}}]})), + FAKE_ENDPOINT, + "key", + nb.RequestBudget(100), + 2, + lambda: next(timestamps), + lambda: 0.0, + ) + assert {row["discovered_at_unix"] for row in results} == {1234.0} + + +def test_probe_models_rejects_incomplete_probe_budget_before_egress() -> None: + """A capability phase never emits biased partial-inventory evidence.""" + models = [ + {"model_id": "a/model-one", "owned_by": ""}, + {"model_id": "b/model-two", "owned_by": ""}, + ] + budget = nb.RequestBudget(5) + calls: list[str] = [] + + def transport( + _method: str, + _url: str, + _headers: dict[str, str], + _body: bytes | None, + ) -> tuple[int, bytes]: + calls.append("called") + return _ok_json({"choices": [{"message": {"content": "OK"}}]}) + + with pytest.raises(nb.BenchmarkBudgetError, match="capability probe plan needs 18"): + nb.probe_discovered_models( + models, + transport, + FAKE_ENDPOINT, + "key", + budget, + 1, + lambda: 1234.0, + lambda: 0.0, + ) + + assert calls == [] + assert budget.requests_spent == 0 + + +def test_probe_models_stop_scheduling_requests_after_auth_rejection() -> None: + calls = 0 + lock = threading.Lock() + + def rejected_transport(method, url, headers, body): + nonlocal calls + with lock: + calls += 1 + return 401, b"{}" + + concurrency = 3 + with pytest.raises(nb.BenchmarkAuthError): + nb.probe_discovered_models( + [ + {"model_id": f"vendor/model-{index}", "owned_by": ""} + for index in range(20) + ], + rejected_transport, + FAKE_ENDPOINT, + "rejected-key", + nb.RequestBudget(500), + concurrency, + lambda: 1234.0, + ) + + assert calls <= concurrency + + +# -------------------------------------------------------------------------- +# Scorers, manifest, pricing +# -------------------------------------------------------------------------- + + +def test_scorers_match_and_miss() -> None: + assert nb.score_exact_number_match({"number": "21"}, "the answer is 21.") == 1.0 + assert nb.score_exact_number_match({"number": "21"}, "the answer is 210") == 0.0 + assert nb.score_exact_number_match({"number": "21"}, "the answer is 21.5") == 0.0 + assert nb.score_exact_number_match({"number": "21"}, "the answer is 121") == 0.0 + assert ( + nb.score_exact_number_match({"number": "0.05"}, "It costs $0.05 total") == 1.0 + ) + assert nb.score_exact_number_match({"number": "21"}, "the answer is -21") == 0.0 + assert nb.score_exact_number_match({"number": "-21"}, "the answer is -21") == 1.0 + assert nb.score_substring_match({"substring": "Paris"}, "It is PARIS indeed") == 1.0 + assert nb.score_substring_match({"substring": "Paris"}, "It is Lyon") == 0.0 + assert ( + nb.score_substring_match( + {"substring": "Au", "strict_case_sensitive": True}, + "Au", + ) + == 1.0 + ) + assert ( + nb.score_substring_match( + {"substring": "Au", "strict_case_sensitive": True}, + "AU", + ) + == 0.0 + ) + assert ( + nb.score_substring_match( + {"substring": "Pacific", "strict_texts": ["Pacific", "Pacific Ocean"]}, + "Pacific Ocean", + ) + == 1.0 + ) + assert ( + nb.score_substring_match( + {"substring": "Pacific", "strict_texts": ["Pacific", "Pacific Ocean"]}, + "The Pacific Ocean", + ) + == 0.0 + ) + + +def _write_json(tmp_path: str, name: str, payload: object) -> str: + path = os.path.join(tmp_path, name) + with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) + return path + + +def test_example_task_manifest_is_valid_and_split() -> None: + manifest = nb.load_task_manifest(TASK_MANIFEST_PATH) + locked = nb.locked_evaluation_tasks(manifest) + assert len(locked) == 30 + assert ( + len(manifest["tasks"]) - len(locked) == 2 + ) # exploratory tuning split stays out + + +def test_manifest_rejects_each_contract_violation() -> None: + with tempfile.TemporaryDirectory() as tmp: + bad_json = os.path.join(tmp, "bad.json") + with open(bad_json, "w", encoding="utf-8") as handle: + handle.write("{broken") + cases = [ + (bad_json, None), + (_write_json(tmp, "list.json", [1]), None), + (_write_json(tmp, "nover.json", {"tasks": []}), None), + ( + _write_json( + tmp, "notasks.json", {"manifest_version": "1", "tasks": []} + ), + None, + ), + ( + _write_json( + tmp, "taskstr.json", {"manifest_version": "1", "tasks": ["x"]} + ), + None, + ), + ] + for path, _ in cases: + with pytest.raises(nb.BenchmarkContractError): + nb.load_task_manifest(path) + + def manifest_with(**overrides: object) -> dict: + task = { + "task_id": "valid_task_one", + "split": "locked", + "prompt": "What color is the clear daytime sky?", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "blue"}, + } + task.update(overrides) + return {"manifest_version": "1", "tasks": [task]} + + violations = [ + manifest_with(task_id="Bad-Id"), + manifest_with(task_id="single"), + manifest_with(split="training"), + manifest_with(prompt=" "), + manifest_with(prompt=42), + manifest_with(scorer="substring_match"), + manifest_with(scorer={"name": "unknown_scorer", "version": "9"}), + manifest_with(expected={}), + manifest_with(expected="blue"), + manifest_with( + scorer={"name": "exact_number_match", "version": "1"}, + expected={"wrong": 21}, + ), + manifest_with( + scorer={"name": "exact_number_match", "version": "1"}, + expected={"number": "NaN"}, + ), + manifest_with(expected={"wrong": "blue"}), + # Leakage: the scorer would award the prompt itself a point. + manifest_with(prompt="Answer blue if the sky is blue."), + manifest_with( + prompt="Answer Pacific Ocean only.", + expected={ + "substring": "Pacific", + "strict_texts": ["Pacific", "Pacific Ocean"], + }, + ), + ] + for index, payload in enumerate(violations): + path = _write_json(tmp, f"violation_{index}.json", payload) + with pytest.raises(nb.BenchmarkContractError): + nb.load_task_manifest(path) + + duplicate = manifest_with() + duplicate["tasks"] = [duplicate["tasks"][0], dict(duplicate["tasks"][0])] + path = _write_json(tmp, "duplicate.json", duplicate) + with pytest.raises(nb.BenchmarkContractError): + nb.load_task_manifest(path) + + +def test_example_pricing_scenario_is_valid_and_none_passthrough() -> None: + scenario = nb.load_pricing_scenario(PRICING_SCENARIO_PATH) + assert scenario["scenario_status"] == "example_unreviewed" + assert nb.load_pricing_scenario(None) is None + + +def test_pricing_scenario_rejects_each_contract_violation() -> None: + with tempfile.TemporaryDirectory() as tmp: + bad_json = os.path.join(tmp, "bad.json") + with open(bad_json, "w", encoding="utf-8") as handle: + handle.write("{broken") + base = { + "scenario_version": "1", + "scenario_status": "reviewed", + "usd_per_million_tokens": {"vendor/model-a": {"input": 1.0, "output": 2.0}}, + } + violations = [ + dict(base, scenario_version=3), + dict(base, scenario_status="draft"), + dict(base, usd_per_million_tokens=[1]), + dict(base, usd_per_million_tokens={"vendor/model-a": "cheap"}), + dict( + base, + usd_per_million_tokens={ + "vendor/model-a": {"input": True, "output": 2.0} + }, + ), + dict( + base, + usd_per_million_tokens={ + "vendor/model-a": {"input": 1.0, "output": "two"} + }, + ), + dict( + base, + usd_per_million_tokens={ + "vendor/model-a": {"input": float("nan"), "output": 2.0} + }, + ), + dict( + base, + usd_per_million_tokens={ + "vendor/model-a": {"input": float("inf"), "output": 2.0} + }, + ), + dict( + base, + usd_per_million_tokens={ + "vendor/model-a": {"input": -0.1, "output": 2.0} + }, + ), + dict(base, usd_per_million_tokens={"vendor/model-a": {"output": 2.0}}), + ] + with pytest.raises(nb.BenchmarkContractError): + nb.load_pricing_scenario(bad_json) + for index, payload in enumerate(violations): + path = _write_json(tmp, f"pricing_{index}.json", payload) + with pytest.raises(nb.BenchmarkContractError): + nb.load_pricing_scenario(path) + + +def test_hypothetical_cost_paths() -> None: + scenario = { + "scenario_version": "1", + "scenario_status": "reviewed", + "usd_per_million_tokens": {"vendor/model-a": {"input": 1.0, "output": 2.0}}, + } + usage = { + "vendor/model-a": {"prompt_tokens": 1_000_000, "completion_tokens": 500_000} + } + assert nb.hypothetical_cost_usd(scenario, usage) == 2.0 + assert nb.hypothetical_cost_usd(None, usage) == "unknown" + unpriced = {"vendor/other-model": {"prompt_tokens": 10, "completion_tokens": 10}} + assert nb.hypothetical_cost_usd(scenario, unpriced) == "unknown" + + +# -------------------------------------------------------------------------- +# Worker pool + usage accounting +# -------------------------------------------------------------------------- + + +def test_sanitize_worker_agent_id_paths() -> None: + taken: set[str] = set() + assert ( + nb.sanitize_worker_agent_id("meta/llama-3.1-8b", taken) == "meta_llama_3_1_8b" + ) + assert ( + nb.sanitize_worker_agent_id("meta/llama-3.1-8b", taken) == "meta_llama_3_1_8b_2" + ) + assert ( + nb.sanitize_worker_agent_id("meta/llama-3.1-8b", taken) == "meta_llama_3_1_8b_3" + ) + assert nb.sanitize_worker_agent_id("gpt", taken) == "nim_gpt" + assert nb.sanitize_worker_agent_id("///", taken) == "unnamed_model" + + +def _probed(model_id: str, chat_eligible: bool = True) -> dict: + return { + "model_id": model_id, + "owned_by": "vendor", + "chat_eligible": chat_eligible, + "model_classification": "chat_capable" if chat_eligible else "embedding_only", + } + + +def test_build_worker_agents_filters_caps_and_validates() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.build_worker_agents([], "mock://x", 0) + probed = [ + _probed("a/chat-one"), + _probed("b/embed-only", chat_eligible=False), + _probed("c/chat-two"), + _probed("d/chat-three"), + ] + agents = nb.build_worker_agents(probed, "mock://x", 2) + assert [agent.model for agent in agents] == ["a/chat-one", "c/chat-two"] + assert all(agent.credential_key == nb.NIM_CREDENTIAL_NAME for agent in agents) + + +def test_token_count_coercion_guards_non_finite_values() -> None: + assert nb._coerce_token_count(7) == 7 + assert nb._coerce_token_count(7.9) == 7 + assert nb._coerce_token_count(True) is None + assert nb._coerce_token_count("7") is None + assert nb._coerce_token_count(float("nan")) is None + assert nb._coerce_token_count(float("inf")) is None + assert nb._coerce_token_count(-1) is None + assert nb._coerce_token_count(None) is None + + +def test_cell_usage_reported_vs_estimated_and_failover() -> None: + agents_by_id = {"worker_one": "vendor/model-a", "worker_two": "vendor/model-b"} + reported_trace = [ + { + "id": 0, + "role": "worker", + "agent_id": "worker_one", + "output": "x", + "usage": {"prompt_tokens": 10, "completion_tokens": 5}, + }, + { + "id": 1, + "role": "verifier", + "agent_id": "worker_one", + "served_agent_id": "worker_two", + "output": "y", + "usage": {"prompt_tokens": 3, "completion_tokens": 2}, + }, + ] + usage_by_model, summary = nb._cell_usage( + reported_trace, agents_by_id, "prompt text" + ) + assert summary["token_usage_source"] == "reported" + assert usage_by_model["vendor/model-a"] == { + "prompt_tokens": 10, + "completion_tokens": 5, + } + assert usage_by_model["vendor/model-b"] == { + "prompt_tokens": 3, + "completion_tokens": 2, + } + assert summary["total_tokens"] == 20 + assert summary["models_used"][1]["agent_id"] == "worker_two" + + adversarial_trace = [ + { + "id": 0, + "role": "worker", + "agent_id": "worker_one", + "output": "answer text", + "usage": {"prompt_tokens": float("nan"), "completion_tokens": float("inf")}, + }, + { + "id": 1, + "role": "worker", + "agent_id": "worker_one", + "output": None, + "usage": "corrupted", + }, + ] + _usage, summary = nb._cell_usage(adversarial_trace, agents_by_id, "prompt text") + assert summary["token_usage_source"] == "estimated" + assert summary["total_tokens"] > 0 + + +def test_cell_usage_rejects_unknown_agent_as_contract_error() -> None: + trace = [ + { + "id": 0, + "role": "worker", + "agent_id": "worker_missing", + "output": "answer", + } + ] + with pytest.raises(nb.BenchmarkContractError, match="unknown agent"): + nb._cell_usage(trace, {"worker_one": "vendor/model-a"}, "prompt text") + + +def test_run_error_classification() -> None: + assert nb._classify_run_error(TimeoutError("slow")) == "timeout" + wrapped = RuntimeError("provider failed") + wrapped.__cause__ = socket.timeout("slow") + assert nb._classify_run_error(wrapped) == "timeout" + assert nb._classify_run_error(ValueError("bad")) == "failure" + + +def _task(task_id: str = "sample_task", expected: str = "zebra") -> dict: + return { + "task_id": task_id, + "split": "locked", + "prompt": "Where do stripes live?", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": expected}, + } + + +def test_run_policy_cell_success_failure_timeout_and_fail_closed() -> None: + agents_by_id = {"worker_one": "vendor/model-a"} + ok = nb.run_policy_cell( + "route_once", + _task(), + lambda: { + "answer": "a zebra appears", + "trace": [ + { + "id": 0, + "role": "worker", + "agent_id": "worker_one", + "output": "a zebra appears", + } + ], + }, + agents_by_id, + None, + nb._deterministic_timer(), + ) + assert ok["run_outcome"] == "success" and ok["task_score"] == 1.0 + assert ok["hypothetical_cost_usd"] == "unknown" and ok["actual_cost_usd"] == 0.0 + assert ok["response_sha256"] and ok["call_count"] == 1 + + def fail() -> dict: + raise RuntimeError("boom") + + failed = nb.run_policy_cell( + "route_once", _task(), fail, agents_by_id, None, nb._deterministic_timer() + ) + assert failed["run_outcome"] == "failure" and failed["task_score"] is None + assert failed["outcome_reason"] == "RuntimeError:boom" + + def provider_failure() -> dict: + raise urllib.error.HTTPError("https://provider", 503, "down", {}, None) + + categorized = nb.run_policy_cell( + "route_once", + _task(), + provider_failure, + agents_by_id, + None, + nb._deterministic_timer(), + ) + assert categorized["outcome_reason"] == "provider_http_error:503" + assert ( + nb._run_error_reason(nb.PolicyTokenBudgetExceeded("limit")) + == "policy_token_budget_exceeded" + ) + + incurred = nb.run_policy_cell( + "route_once", + _task(), + fail, + agents_by_id, + None, + nb._deterministic_timer(), + lambda: { + "call_count": 1, + "prompt_tokens": 7, + "completion_tokens": 5, + "total_tokens": 12, + "models_used": [{"model_id": "vendor/model-a"}], + }, + ) + assert incurred["call_count"] == 1 and incurred["total_tokens"] == 12 + assert incurred["prompt_tokens"] == 7 + assert incurred["completion_tokens"] == 5 + assert incurred["models_used"] == [{"model_id": "vendor/model-a"}] + + def slow() -> dict: + raise TimeoutError("deadline") + + timed_out = nb.run_policy_cell( + "route_once", _task(), slow, agents_by_id, None, nb._deterministic_timer() + ) + assert timed_out["run_outcome"] == "timeout" + + def out_of_budget() -> dict: + raise nb.BenchmarkBudgetError("exhausted") + + with pytest.raises(nb.BenchmarkBudgetError): + nb.run_policy_cell( + "route_once", + _task(), + out_of_budget, + agents_by_id, + None, + nb._deterministic_timer(), + ) + + with pytest.raises(nb.BenchmarkContractError): + nb.run_policy_cell( + "route_once", + _task(), + lambda: (_ for _ in ()).throw(nb.BenchmarkContractError("transport")), + agents_by_id, + None, + nb._deterministic_timer(), + ) + + +def test_cheapest_priced_agent_selection() -> None: + agents = _mock_agents("vendor/model-a", "vendor/model-b", "vendor/model-c") + scenario = { + "scenario_version": "1", + "scenario_status": "reviewed", + "usd_per_million_tokens": { + "vendor/model-b": {"input": 0.1, "output": 0.2}, + "vendor/model-c": {"input": 0.1, "output": 0.2}, + }, + } + assert nb.cheapest_priced_agent(agents, None) is None + assert ( + nb.cheapest_priced_agent( + agents, + { + "scenario_version": "1", + "scenario_status": "reviewed", + "usd_per_million_tokens": {}, + }, + ) + is None + ) + # Deterministic tiebreak: equal combined rate resolves by model id. + assert nb.cheapest_priced_agent(agents, scenario).model == "vendor/model-b" + + +def test_planned_evaluation_requests_formula() -> None: + assert nb.planned_evaluation_requests(3, 10) == 10 * ( + 3 * 2 + nb.MAX_WORKFLOW_DEPTH + nb.MAX_WORKFLOW_DEPTH + 2 + ) + + +def test_evaluate_policies_contract_failures() -> None: + client = ModelClient() + with pytest.raises(nb.BenchmarkContractError): + nb.evaluate_policies([], _mini_manifest(), None, client, nb.RequestBudget(100)) + agents = _mock_agents("vendor/model-a") + exploratory_only = { + "manifest_version": "1", + "tasks": [dict(_task(), split="exploratory")], + } + with pytest.raises(nb.BenchmarkContractError): + nb.evaluate_policies( + agents, exploratory_only, None, client, nb.RequestBudget(100) + ) + with pytest.raises(nb.BenchmarkBudgetError): + nb.evaluate_policies( + agents, _mini_manifest(), None, client, nb.RequestBudget(2) + ) + + class RememberedContractClient(ModelClient): + benchmark_contract_error = nb.BenchmarkContractError("transport contract") + + def chat(self, *args, **kwargs): + return "apparently successful fallback" + + with pytest.raises(nb.BenchmarkContractError, match="transport contract"): + nb.evaluate_policies( + agents, + _mini_manifest(), + None, + RememberedContractClient(), + nb.RequestBudget(100), + ) + + +def test_evaluate_policies_all_arms_with_pricing() -> None: + agents = _mock_agents("dryrun/chat-basic", "dryrun/chat-vision") + scenario = nb.load_pricing_scenario(PRICING_SCENARIO_PATH) + budget = nb.RequestBudget(200) + evaluation = nb.evaluate_policies( + agents, + _mini_manifest(3), + scenario, + nb._BudgetedModelClient(budget), + budget, + nb._deterministic_timer(), + ) + cells = evaluation["evaluation_cells"] + policies = {cell["policy_name"] for cell in cells} + assert policies == { + "direct_single_worker:dryrun/chat-basic", + "direct_single_worker:dryrun/chat-vision", + "route_once", + "conduct_bounded", + "cheapest_eligible_worker", + } + assert evaluation["cheapest_worker_skip_reason"] is None + conduct_cells = [cell for cell in cells if cell["policy_name"] == "conduct_bounded"] + assert all( + cell["workflow_depth"] <= nb.MAX_WORKFLOW_DEPTH for cell in conduct_cells + ) + assert all( + cell["configured_total_token_budget"] == nb.DEFAULT_POLICY_TOTAL_TOKEN_BUDGET + for cell in conduct_cells + ) + assert all( + cell["configured_maximum_calls"] == nb.MAX_WORKFLOW_DEPTH + for cell in conduct_cells + ) + assert all( + cell["observed_budget_calls"] <= nb.MAX_WORKFLOW_DEPTH for cell in conduct_cells + ) + assert all(cell["run_outcome"] == "success" for cell in conduct_cells) + assert cells == sorted( + cells, key=lambda cell: (cell["policy_name"], cell["task_id"]) + ) + assert budget.requests_spent > 0 + + +def test_evaluate_policies_preserves_reported_usage_source() -> None: + class ReportedUsageClient(ModelClient): + def chat(self, *args, **kwargs): + return "zebra" + + def take_usage(self): + return {"prompt_tokens": 2, "completion_tokens": 1} + + evaluation = nb.evaluate_policies( + _mock_agents("dryrun/chat-basic"), + _mini_manifest(1), + None, + ReportedUsageClient(), + nb.RequestBudget(100), + ) + assert any( + cell["token_usage_source"] == "reported" + for cell in evaluation["evaluation_cells"] + ) + + +def test_equal_budget_take_usage_reconciles_prompt_and_completion_independently() -> None: + """A small reported prompt cannot turn failed-cell completion usage negative.""" + + class ReportedUsageClient(ModelClient): + def chat(self, *args, **kwargs): + return "estimated completion is deliberately longer" + + def take_usage(self): + return {"prompt_tokens": 1, "completion_tokens": 0} + + agent = _mock_agents("dryrun/chat-basic")[0] + cell = nb.EqualBudgetModelClient( + ReportedUsageClient(), total_token_budget=100, maximum_calls=1 + ) + cell.chat(agent, [{"role": "user", "content": "a deliberately long prompt"}]) + cell.take_usage() + + assert cell.observed_prompt_tokens == 1 + assert cell.observed_completion_tokens == 0 + assert cell.observed_tokens == 1 + assert cell.estimated_usage_by_model[agent.model] == { + "prompt_tokens": 1, + "completion_tokens": 0, + } + + +def test_evaluate_policies_records_observed_budget_overflow() -> None: + class OversizedAnswerClient(ModelClient): + def chat(self, *args, **kwargs) -> str: # type: ignore[override] + del args, kwargs + return "x" * 5000 + + evaluation = nb.evaluate_policies( + _mock_agents("dryrun/chat-basic"), + _mini_manifest(1), + None, + OversizedAnswerClient(), + nb.RequestBudget(100), + total_token_budget=512, + ) + + assert evaluation["evaluation_cells"] + assert all( + cell["outcome_reason"] == "observed_usage_exceeded_equal_token_budget" + for cell in evaluation["evaluation_cells"] + ) + + +def test_evaluate_policies_skip_reasons_without_pricing() -> None: + agents = _mock_agents("vendor/model-a") + budget = nb.RequestBudget(200) + evaluation = nb.evaluate_policies( + agents, _mini_manifest(), None, ModelClient(), budget + ) + assert evaluation["cheapest_worker_skip_reason"] == "no_pricing_scenario_supplied" + unpriced_scenario = { + "scenario_version": "1", + "scenario_status": "reviewed", + "usd_per_million_tokens": {"vendor/other": {"input": 1.0, "output": 1.0}}, + } + evaluation = nb.evaluate_policies( + agents, + _mini_manifest(), + unpriced_scenario, + ModelClient(), + nb.RequestBudget(200), + ) + assert evaluation["cheapest_worker_skip_reason"] == "no_worker_priced_by_scenario" + + +# -------------------------------------------------------------------------- +# Statistics +# -------------------------------------------------------------------------- + + +def test_paired_bootstrap_requires_pairs_and_is_deterministic() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.paired_bootstrap_mean_difference([]) + first = nb.paired_bootstrap_mean_difference( + [(1.0, 0.0), (0.5, 0.5), (1.0, 0.5)], seed=11 + ) + second = nb.paired_bootstrap_mean_difference( + [(1.0, 0.0), (0.5, 0.5), (1.0, 0.5)], seed=11 + ) + assert first == second + assert first["ci_low"] <= first["mean_difference"] <= first["ci_high"] + assert first["pair_count"] == 3 + + +def test_pareto_frontier_excludes_dominated_rows() -> None: + rows = [ + {"name": "good_cheap", "quality": 0.9, "cost": 1.0}, + {"name": "good_pricey", "quality": 0.9, "cost": 2.0}, + {"name": "bad_cheap", "quality": 0.1, "cost": 0.5}, + {"name": "bad_pricey", "quality": 0.1, "cost": 5.0}, + ] + frontier = nb.pareto_frontier(rows, "quality", "cost") + assert [row["name"] for row in frontier] == ["good_cheap", "bad_cheap"] + + +def _synthetic_cell( + policy: str, task_id: str, score, outcome: str = "success", cost=0.5 +) -> dict: + return { + "policy_name": policy, + "task_id": task_id, + "task_split": "locked", + "scorer_name": "substring_match", + "scorer_version": "1", + "task_score": score, + "run_outcome": outcome, + "outcome_reason": "completed", + "end_to_end_latency_ms": 10.0, + "provider_latency_ms": None, + "call_count": 1, + "workflow_depth": 1, + "prompt_tokens": 4, + "completion_tokens": 4, + "total_tokens": 8, + "token_usage_source": "estimated", + "actual_cost_usd": 0.0, + "hypothetical_cost_usd": cost, + "models_used": [], + "response_sha256": "hash", + } + + +def test_summaries_label_unknown_costs_and_all_failure_policies() -> None: + cells = [ + _synthetic_cell("route_once", "task_one", 1.0, cost=0.5), + _synthetic_cell("route_once", "task_two", 0.0, cost="unknown"), + _synthetic_cell( + "broken_policy", "task_one", None, outcome="failure", cost="unknown" + ), + ] + summaries = {row["policy_name"]: row for row in nb.summarize_policies(cells)} + assert summaries["route_once"]["mean_task_score"] == 0.5 + assert summaries["route_once"]["mean_hypothetical_cost_usd"] == "unknown" + assert summaries["route_once"]["unknown_hypothetical_cost_cells"] == 1 + assert summaries["broken_policy"]["mean_task_score"] == 0.0 + assert summaries["broken_policy"]["mean_hypothetical_cost_usd"] == "unknown" + assert summaries["broken_policy"]["success_count"] == 0 + assert summaries["broken_policy"]["completion_fraction"] == 0.0 + + +def test_summaries_count_failures_as_zero_quality() -> None: + summaries = nb.summarize_policies( + [ + _synthetic_cell("flaky_policy", "task_one", 1.0), + _synthetic_cell( + "flaky_policy", "task_two", None, outcome="failure", cost="unknown" + ), + ] + ) + assert summaries[0]["mean_task_score"] == 0.5 + assert summaries[0]["completion_fraction"] == 0.5 + assert nb.build_pareto_frontiers(summaries)["quality_vs_hypothetical_cost"] == [] + + +def test_optional_cheapest_policy_does_not_change_evidence_completion() -> None: + cells = [ + _synthetic_cell("route_once", "task_one", 1.0), + _synthetic_cell("conduct_bounded", "task_one", 1.0), + _synthetic_cell( + "cheapest_eligible_worker", "task_one", None, outcome="failure" + ), + ] + summary = nb._evaluation_evidence_summary(cells, nb.MINIMUM_PAIRED_TASK_COUNT) + assert summary["observed_completion_fraction"] == 1.0 + + +def test_best_single_worker_hindsight_selection() -> None: + assert ( + nb.best_single_worker_hindsight( + [{"policy_name": "route_once", "mean_task_score": 1.0}] + ) + is None + ) + summaries = nb.summarize_policies( + [ + _synthetic_cell("direct_single_worker:vendor/model-a", "task_one", 0.0), + _synthetic_cell("direct_single_worker:vendor/model-b", "task_one", 1.0), + ] + ) + best = nb.best_single_worker_hindsight(summaries) + assert best["model_id"] == "vendor/model-b" + assert best["selection_basis"] == "hindsight_argmax_mean_locked_score" + + +def test_paired_policy_comparisons_skip_missing_and_disjoint() -> None: + disjoint = [ + _synthetic_cell("conduct_bounded", "task_one", 1.0), + _synthetic_cell("route_once", "task_two", 0.0), + ] + assert nb.paired_policy_comparisons(disjoint, seed=3) == [] + cells = [ + _synthetic_cell("conduct_bounded", "task_one", 1.0), + _synthetic_cell("route_once", "task_one", 0.0), + _synthetic_cell("direct_single_worker:vendor/model-a", "task_one", 1.0), + # Failed cells carry no score and must stay out of the pairing. + _synthetic_cell("route_once", "task_three", None, outcome="failure"), + ] + comparisons = nb.paired_policy_comparisons(cells, seed=3) + pairs = {(row["policy_a"], row["policy_b"]) for row in comparisons} + assert ("conduct_bounded", "route_once") in pairs + assert ("route_once", "direct_single_worker:vendor/model-a") in pairs + + +def test_pareto_frontiers_exclude_unknown_cost_policies() -> None: + summaries = nb.summarize_policies( + [ + _synthetic_cell("route_once", "task_one", 1.0, cost=0.5), + _synthetic_cell("conduct_bounded", "task_one", 1.0, cost="unknown"), + ] + ) + frontiers = nb.build_pareto_frontiers(summaries) + assert [ + row["policy_name"] for row in frontiers["quality_vs_hypothetical_cost"] + ] == ["route_once"] + assert frontiers["excluded_unknown_cost_policies"] == ["conduct_bounded"] + assert len(frontiers["quality_vs_latency"]) >= 1 + + +# -------------------------------------------------------------------------- +# Provenance, schema, artifacts, secrets +# -------------------------------------------------------------------------- + + +def test_hash_helpers_are_stable() -> None: + assert nb.sha256_of_json({"b": 1, "a": 2}) == nb.sha256_of_json({"a": 2, "b": 1}) + assert len(nb.sha256_of_file(TASK_MANIFEST_PATH)) == 64 + + +def test_provenance_fails_closed_for_live_without_identity() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.build_provenance("live", "", "", {}, TASK_MANIFEST_PATH, None, {}) + live = nb.build_provenance( + "live", + "a" * 40, + "run-9", + {}, + TASK_MANIFEST_PATH, + PRICING_SCENARIO_PATH, + {"seed": 7}, + ) + assert live["pricing_scenario_sha256"] is not None + for invalid_sha in ("abc123", "g" * 40, "a" * 39, "A" * 40): + with pytest.raises(nb.BenchmarkContractError, match="valid --git-sha"): + nb.build_provenance( + "live", invalid_sha, "run-9", {}, TASK_MANIFEST_PATH, None, {} + ) + dry = nb.build_provenance("dry_run", "", "", {}, TASK_MANIFEST_PATH, None, {}) + assert dry["git_sha"] == nb.DRY_RUN_PROVENANCE_PLACEHOLDER + assert dry["pricing_scenario_sha256"] is None + + +def test_report_schema_validation_reports_missing_paths() -> None: + with pytest.raises(nb.BenchmarkContractError) as excinfo: + nb.validate_report_schema({"provenance": "not-a-dict"}) + assert "provenance.run_mode" in str(excinfo.value) + + +def _dry_report(output_dir: str) -> dict: + return nb.run_benchmark( + "dry_run", + TASK_MANIFEST_PATH, + PRICING_SCENARIO_PATH, + output_dir, + max_total_requests=900, + ) + + +def test_evaluation_contract_failure_publishes_no_artifacts( + tmp_path: Path, +) -> None: + register_credential(nb.NIM_CREDENTIAL_NAME, "nvapi-test-credential") + dry_transport = nb.build_dry_run_transport() + _, catalog_body = dry_transport( + "GET", f"{FAKE_ENDPOINT}/models", {}, None + ) + discovered_count = len(nb.parse_model_catalog_body(catalog_body)["models"]) + probe_phase_calls = 1 + discovered_count * len(nb.CAPABILITY_PROBE_ORDER) + calls = 0 + + def malformed_during_evaluation(method, url, headers, body): + nonlocal calls + calls += 1 + if calls <= probe_phase_calls: + return dry_transport(method, url, headers, body) + if calls == probe_phase_calls + 1: + return 200, b"[]" + return dry_transport(method, url, headers, body) + + with pytest.raises(nb.BenchmarkContractError, match="must be an object"): + nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + None, + str(tmp_path), + max_total_requests=900, + git_sha="e" * 40, + workflow_run_id="run-contract-failure", + transport=malformed_during_evaluation, + ) + + assert list(tmp_path.iterdir()) == [] + + +def test_artifact_writer_refuses_secret_leak() -> None: + register_credential(nb.NIM_CREDENTIAL_NAME, "nvapi-super-secret-value") + with tempfile.TemporaryDirectory() as tmp: + report = _dry_report(os.path.join(tmp, "clean")) + # The honest artifacts never contain the credential... + serialized = json.dumps(report) + assert "nvapi-super-secret-value" not in serialized + # ...and a poisoned report is refused outright. + report["catalog_snapshot"]["probed_models"][0]["owned_by"] = ( + "nvapi-super-secret-value" + ) + with pytest.raises(nb.SecretLeakError): + nb.write_benchmark_artifacts(report, os.path.join(tmp, "leaky")) + + +def test_artifact_writer_uses_shared_four_file_publication() -> None: + with tempfile.TemporaryDirectory() as tmp: + target = os.path.join(tmp, "evidence") + report = _dry_report(target) + assert set(os.listdir(target)) == { + "benchmark_report.json", + "benchmark_cells.csv", + "benchmark_summary.md", + "run_provenance.json", + } + provenance = json.loads(Path(target, "run_provenance.json").read_text()) + assert provenance["catalog_snapshot_sha256"] == report["provenance"][ + "catalog_snapshot_sha256" + ] + + +def test_secret_guard_passes_when_no_secret_registered() -> None: + nb._ensure_secret_absent("no secret registered anywhere") + + +# -------------------------------------------------------------------------- +# Dry-run provider + full pipeline +# -------------------------------------------------------------------------- + + +def test_dry_run_transport_serves_all_paths() -> None: + transport = nb.build_dry_run_transport() + status, body = transport("GET", f"{FAKE_ENDPOINT}/models", {}, None) + assert status == 200 and b"dryrun/chat-omni" in body + status, _ = transport( + "POST", + f"{FAKE_ENDPOINT}/chat/completions", + {}, + b'{"model": "dryrun/unknown-model"}', + ) + assert status == 404 + status, _ = transport( + "POST", f"{FAKE_ENDPOINT}/chat/completions", {}, b"no model marker at all" + ) + assert status == 404 + status, _ = transport( + "POST", + f"{FAKE_ENDPOINT}/chat/completions", + {}, + b'{"model": "dryrun/throttled-model"}', + ) + assert status == 429 + status, _ = transport( + "POST", + f"{FAKE_ENDPOINT}/chat/completions", + {}, + b'{"model": "dryrun/outage-model"}', + ) + assert status == 503 + status, _ = transport( + "POST", + f"{FAKE_ENDPOINT}/chat/completions", + {}, + b'{"model": "dryrun/legacy-unsupported"}', + ) + assert status == 404 + status, _ = transport( + "POST", f"{FAKE_ENDPOINT}/embeddings", {}, b'{"model": "dryrun/chat-basic"}' + ) + assert status == 400 + status, body = transport( + "POST", f"{FAKE_ENDPOINT}/embeddings", {}, b'{"model": "dryrun/embed-basic"}' + ) + assert status == 200 and b"embedding" in body + status, body = transport( + "POST", + f"{FAKE_ENDPOINT}/responses", + {}, + b'{"model": "dryrun/responses-native"}', + ) + assert status == 200 and b"output_text" in body + multipart = nb._multipart_transcription_body("dryrun/audio-transcribe") + status, body = transport( + "POST", f"{FAKE_ENDPOINT}/audio/transcriptions", {}, multipart + ) + assert status == 200 and b"text" in body + status, body = transport( + "POST", f"{FAKE_ENDPOINT}/audio/speech", {}, b'{"model": "dryrun/audio-speech"}' + ) + assert status == 200 and body.startswith(b"RIFF") + with pytest.raises(nb.CatalogDiscoveryError): + transport( + "POST", + f"{FAKE_ENDPOINT}/never/heard-of-it", + {}, + b'{"model": "dryrun/chat-basic"}', + ) + + +def test_dry_run_success_bodies_per_endpoint() -> None: + assert b"embedding" in nb._dry_run_success_body("/v1/embeddings") + assert b"output_text" in nb._dry_run_success_body("/v1/responses") + assert b"text" in nb._dry_run_success_body("/v1/audio/transcriptions") + assert nb._dry_run_success_body("/v1/audio/speech").startswith(b"RIFF") + assert b"choices" in nb._dry_run_success_body("/v1/chat/completions") + + +def test_deterministic_timer_advances_monotonically() -> None: + timer = nb._deterministic_timer() + assert timer() < timer() < timer() + + +def test_run_benchmark_rejects_unknown_mode() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.run_benchmark("test", TASK_MANIFEST_PATH, None, "unused") + + +def test_run_benchmark_rejects_output_cap_before_egress() -> None: + calls = 0 + + def transport(*_args) -> tuple[int, bytes]: + nonlocal calls + calls += 1 + return 200, b"{}" + + with pytest.raises(nb.BenchmarkContractError, match="max_output_tokens"): + nb.run_benchmark( + "dry_run", + TASK_MANIFEST_PATH, + None, + "unused", + max_output_tokens=0, + transport=transport, + ) + assert calls == 0 + + +def test_dry_run_pipeline_covers_every_modality_and_is_deterministic() -> None: + with tempfile.TemporaryDirectory() as tmp: + first = _dry_report(os.path.join(tmp, "one")) + second = _dry_report(os.path.join(tmp, "two")) + assert first["capability_summary"] == { + "audio_only": 2, + "chat_capable": 1, + "completion_only": 1, + "embedding_only": 1, + "omni_capable": 1, + "rate_limited": 1, + "responses_only": 1, + "unavailable": 1, + "unsupported_for_contract": 1, + "vision_chat_capable": 2, + } + by_model = { + row["model_id"]: row for row in first["catalog_snapshot"]["probed_models"] + } + assert by_model["dryrun/chat-omni"]["model_classification"] == "omni_capable" + assert set(by_model["dryrun/chat-omni"]["supported_capabilities"]) >= { + "chat_completion", + "image_understanding", + "video_understanding", + "audio_understanding", + } + assert by_model["dryrun/audio-transcribe"]["supported_capabilities"] == [ + "audio_transcription" + ] + assert by_model["dryrun/audio-speech"]["supported_capabilities"] == [ + "audio_speech" + ] + assert ( + by_model["dryrun/embed-basic"]["model_classification"] == "embedding_only" + ) + assert ( + by_model["dryrun/chat-video"]["model_classification"] + == "vision_chat_capable" + ) + # Catalog hygiene lists survive into the snapshot. + assert first["catalog_snapshot"]["duplicate_model_ids"] == ["dryrun/chat-basic"] + assert ( + first["catalog_snapshot"]["invalid_entries"][0]["invalid_reason"] + == "missing_model_id" + ) + # The evaluation compares every required system. + assert first["evaluation"]["best_single_worker_hindsight"] is not None + assert first["evaluation"]["pareto_frontiers"]["quality_vs_latency"] + assert first["evaluation"]["paired_comparisons"] + # Deterministic artifacts: identical reports across runs. + with open(os.path.join(tmp, "one", "benchmark_report.json"), "rb") as handle: + first_bytes = handle.read() + with open(os.path.join(tmp, "two", "benchmark_report.json"), "rb") as handle: + second_bytes = handle.read() + assert first_bytes == second_bytes + assert ( + first["provenance"]["catalog_snapshot_sha256"] + == second["provenance"]["catalog_snapshot_sha256"] + ) + for artifact in ( + "benchmark_report.json", + "benchmark_cells.csv", + "benchmark_summary.md", + ): + assert os.path.exists(os.path.join(tmp, "one", artifact)) + + +def test_dry_run_accepts_explicit_transport() -> None: + with tempfile.TemporaryDirectory() as tmp: + report = nb.run_benchmark( + "dry_run", + TASK_MANIFEST_PATH, + None, + tmp, + max_total_requests=900, + transport=nb.build_dry_run_transport(), + ) + assert report["provenance"]["pricing_scenario_sha256"] is None + assert ( + report["evaluation"]["cheapest_worker_skip_reason"] + == "no_pricing_scenario_supplied" + ) + + +def test_live_run_fails_closed_without_credential() -> None: + with tempfile.TemporaryDirectory() as tmp: + with pytest.raises(NotConfigured): + nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + None, + tmp, + git_sha="a" * 40, + workflow_run_id="run-1", + ) + + +def test_live_run_end_to_end_offline() -> None: + register_credential(nb.NIM_CREDENTIAL_NAME, "nvapi-test-credential") + original_validate = ModelClient._validate_provider + original_send = ModelClient._send + ModelClient._validate_provider = lambda self, agent: None + ModelClient._send = lambda self, agent, payload: "stub live answer" + try: + with tempfile.TemporaryDirectory() as tmp: + report = nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + None, + tmp, + max_total_requests=900, + git_sha="b" * 40, + workflow_run_id="run-42", + transport=nb.build_dry_run_transport(), + ) + finally: + ModelClient._validate_provider = original_validate + ModelClient._send = original_send + assert report["provenance"]["run_mode"] == "live" + assert report["provenance"]["git_sha"] == "b" * 40 + assert report["honesty_labels"]["actual_cost_basis"] == ( + "reviewed_nvidia_developer_program_hosted_endpoint_access" + ) + assert report["request_budget"]["requests_spent"] <= 900 + assert "nvapi-test-credential" not in json.dumps(report) + + +def test_live_run_uses_default_transport_builder_when_none_given() -> None: + register_credential(nb.NIM_CREDENTIAL_NAME, "nvapi-test-credential") + original_builder = nb.build_default_transport + nb.build_default_transport = lambda timeout_seconds: nb.build_dry_run_transport() + original_validate = ModelClient._validate_provider + original_send = ModelClient._send + ModelClient._validate_provider = lambda self, agent: None + ModelClient._send = lambda self, agent, payload: "stub live answer" + try: + with tempfile.TemporaryDirectory() as tmp: + report = nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + None, + tmp, + max_total_requests=900, + git_sha="c" * 40, + workflow_run_id="run-43", + ) + finally: + nb.build_default_transport = original_builder + ModelClient._validate_provider = original_validate + ModelClient._send = original_send + assert report["provenance"]["workflow_run_id"] == "run-43" + + +# -------------------------------------------------------------------------- +# CLI + bootstrap +# -------------------------------------------------------------------------- + + +def test_bootstrap_live_credential_paths() -> None: + from contextual_orchestrator.credentials import get_credential + + nb._bootstrap_live_credential() # neither KV nor env: stays unset + assert get_credential(nb.NIM_CREDENTIAL_NAME) is None + os.environ[nb.NIM_CREDENTIAL_NAME] = "nvapi-from-env" + try: + nb._bootstrap_live_credential() # env seeds the KV (bootstrap transport) + assert get_credential(nb.NIM_CREDENTIAL_NAME) == "nvapi-from-env" + os.environ[nb.NIM_CREDENTIAL_NAME] = "nvapi-different" + nb._bootstrap_live_credential() # existing KV value wins; no re-seed + assert get_credential(nb.NIM_CREDENTIAL_NAME) == "nvapi-from-env" + finally: + os.environ.pop(nb.NIM_CREDENTIAL_NAME, None) + + +def test_cli_dry_run_succeeds() -> None: + with tempfile.TemporaryDirectory() as tmp: + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + exit_code = nb.run_benchmark_cli( + [ + "--dry-run", + "--task-manifest", + TASK_MANIFEST_PATH, + "--pricing-scenario", + PRICING_SCENARIO_PATH, + "--output-dir", + tmp, + "--max-total-requests", + "900", + ] + ) + assert exit_code == 0 + printed = json.loads(stdout.getvalue()) + assert printed["run_mode"] == "dry_run" + assert printed["capability_summary"]["omni_capable"] == 1 + + +def test_cli_fails_closed_on_missing_manifest() -> None: + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + exit_code = nb.run_benchmark_cli( + ["--dry-run", "--task-manifest", "does/not/exist.json"] + ) + assert exit_code == 1 + assert json.loads(stdout.getvalue())["benchmark_failed_closed"] is True + + +def test_cli_live_fails_closed_without_secret() -> None: + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + exit_code = nb.run_benchmark_cli( + [ + "--task-manifest", + TASK_MANIFEST_PATH, + "--git-sha", + "d" * 40, + "--workflow-run-id", + "run-1", + ] + ) + assert exit_code == 1 + assert json.loads(stdout.getvalue())["error_class"] == "NotConfigured" + + +def test_cli_failure_redacts_resolved_bearer(monkeypatch: pytest.MonkeyPatch) -> None: + secret = "nvapi-secret-value-123456" + register_credential(nb.NIM_CREDENTIAL_NAME, secret) + + def fail(*args, **kwargs): + raise nb.BenchmarkContractError(f"Bearer {secret}") + + monkeypatch.setattr(nb, "run_benchmark", fail) + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + exit_code = nb.run_benchmark_cli(["--dry-run"]) + assert exit_code == 1 + assert secret not in stdout.getvalue() + assert "[REDACTED]" in stdout.getvalue() + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/tests/test_nim_benchmark_release_acceptance.py b/tests/test_nim_benchmark_release_acceptance.py new file mode 100644 index 000000000..3171841e6 --- /dev/null +++ b/tests/test_nim_benchmark_release_acceptance.py @@ -0,0 +1,715 @@ +"""Release-level security, fairness, and evidence contracts for the NIM benchmark.""" + +from __future__ import annotations + +import hashlib +import json +from contextlib import contextmanager +from pathlib import Path +import subprocess +import sys +import threading +import urllib.parse + +import pytest + +from contextual_orchestrator import nim_benchmark as nb +from contextual_orchestrator.credentials import ( + InMemoryCredentialBackend, + register_credential, + set_backend, +) +from contextual_orchestrator.orchestrator import ModelClient + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +TASK_MANIFEST_PATH = str(REPOSITORY_ROOT / "examples" / "nim_task_manifest.json") +EXAMPLE_PRICING_PATH = REPOSITORY_ROOT / "examples" / "nim_pricing_scenario.json" +FAKE_ENDPOINT = "https://nim.example.test/v1" + + +@pytest.fixture(autouse=True) +def _isolated_credentials() -> None: + """Give every test a fresh KV backend and remove it after the assertion.""" + set_backend(InMemoryCredentialBackend()) + try: + yield + finally: + set_backend(None) + + +def _write_json(path: Path, payload: object) -> str: + """Write one deterministic JSON fixture and return its string path.""" + path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") + return str(path) + + +def _reviewed_pricing_scenario(**overrides: object) -> dict[str, object]: + """Return a complete reviewed hypothetical-price evidence fixture.""" + scenario: dict[str, object] = { + "scenario_version": "test-reviewed.1", + "scenario_status": "reviewed", + "source_url": "https://pricing.example.test/reviewed-rate-card", + "reviewed_by": "independent_pricing_reviewer", + "reviewed_at_date": "2026-08-05", + "valid_until_date": "2026-09-04", + "rate_basis": "hypothetical_usd_per_million_prompt_and_completion_tokens", + "uncertainty": "Scenario rates are explicit assumptions, not NVIDIA model prices.", + "usd_per_million_tokens": { + "vendor/model-one": {"input": 1.0, "output": 2.0} + }, + } + scenario.update(overrides) + return scenario + + +def _unexpected_transport(*_args: object, **_kwargs: object) -> tuple[int, bytes]: + """Fail a test when validation did not stop before provider egress.""" + raise AssertionError("provider transport must not run before evidence validation") + + +def test_package_import_does_not_eagerly_load_optional_benchmark() -> None: + """Normal gateway imports must not load or mutate the optional evaluator.""" + command = [ + sys.executable, + "-c", + ( + "import sys; import contextual_orchestrator; " + "assert 'contextual_orchestrator.nim_benchmark' not in sys.modules; " + "assert 'contextual_orchestrator.nim_benchmark_hardening' not in sys.modules" + ), + ] + completed = subprocess.run(command, check=False, capture_output=True, text=True) + assert completed.returncode == 0, completed.stderr + + +def test_live_run_rejects_unreviewed_pricing_before_egress(tmp_path: Path) -> None: + """Schema-demo prices can support dry runs but can never drive a live policy.""" + register_credential(nb.NIM_CREDENTIAL_NAME, "secret-test-key") + scenario = json.loads(EXAMPLE_PRICING_PATH.read_text(encoding="utf-8")) + scenario_path = _write_json(tmp_path / "unreviewed_pricing.json", scenario) + + with pytest.raises(nb.BenchmarkContractError, match="reviewed"): + nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + scenario_path, + str(tmp_path / "artifacts"), + endpoint=FAKE_ENDPOINT, + git_sha="a" * 40, + workflow_run_id="123", + transport=_unexpected_transport, + ) + + +def test_live_run_rejects_incomplete_or_expired_pricing_before_egress( + tmp_path: Path, +) -> None: + """Live hypothetical prices need complete, current, independently reviewed evidence.""" + register_credential(nb.NIM_CREDENTIAL_NAME, "secret-test-key") + incomplete = _reviewed_pricing_scenario() + del incomplete["reviewed_by"] + incomplete_path = _write_json(tmp_path / "incomplete_pricing.json", incomplete) + with pytest.raises(nb.BenchmarkContractError, match="reviewed_by"): + nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + incomplete_path, + str(tmp_path / "incomplete_artifacts"), + endpoint=FAKE_ENDPOINT, + git_sha="b" * 40, + workflow_run_id="124", + transport=_unexpected_transport, + ) + + expired_path = _write_json( + tmp_path / "expired_pricing.json", + _reviewed_pricing_scenario( + reviewed_at_date="1999-01-01", valid_until_date="2000-01-01" + ), + ) + with pytest.raises(nb.BenchmarkContractError, match="expired"): + nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + expired_path, + str(tmp_path / "expired_artifacts"), + endpoint=FAKE_ENDPOINT, + git_sha="c" * 40, + workflow_run_id="125", + transport=_unexpected_transport, + ) + + +def test_probe_concurrency_executes_the_complete_cartesian_plan() -> None: + """Thread scheduling cannot turn a complete probe plan into a biased prefix.""" + models = [ + {"model_id": "a/model-one", "owned_by": "vendor"}, + {"model_id": "b/model-two", "owned_by": "vendor"}, + ] + first_model_started = threading.Event() + second_model_four_calls = threading.Event() + second_model_call_count = 0 + count_lock = threading.Lock() + + def transport( + _method: str, + _url: str, + _headers: dict[str, str], + body: bytes | None, + ) -> tuple[int, bytes]: + """Force completion-order drift while every preflighted cell still runs.""" + nonlocal second_model_call_count + payload = (body or b"").decode("utf-8", errors="ignore") + if "a/model-one" in payload: + first_model_started.set() + second_model_four_calls.wait(timeout=0.25) + elif "b/model-two" in payload: + first_model_started.wait(timeout=0.25) + with count_lock: + second_model_call_count += 1 + if second_model_call_count == 4: + second_model_four_calls.set() + return 400, b"{}" + + budget = nb.RequestBudget( + len(models) * len(nb.CAPABILITY_PROBE_ORDER) + ) + rows = nb.probe_discovered_models( + models, + transport, + FAKE_ENDPOINT, + "credential-redacted", + budget, + probe_concurrency=2, + clock=lambda: 1.0, + timer=lambda: 0.0, + ) + + assert [row["model_id"] for row in rows] == ["a/model-one", "b/model-two"] + for model_row in rows: + assert [ + probe_row["capability_name"] + for probe_row in model_row["capability_probe_rows"] + ] == list(nb.CAPABILITY_PROBE_ORDER) + assert all( + probe_row["probe_outcome"] != "skipped" + for probe_row in model_row["capability_probe_rows"] + ) + assert budget.requests_spent == 18 + + +def test_complete_request_plan_rejects_invalid_counts() -> None: + """Planning inputs are positive integers, never booleans or empty counts.""" + invalid_cases = [ + {"discovered_model_count": 0, "max_eval_models": 7, "locked_task_count": 10}, + {"discovered_model_count": True, "max_eval_models": 7, "locked_task_count": 10}, + {"discovered_model_count": 1, "max_eval_models": 0, "locked_task_count": 10}, + {"discovered_model_count": 1, "max_eval_models": 7, "locked_task_count": 0}, + ] + + for case in invalid_cases: + with pytest.raises(nb.BenchmarkContractError, match="positive integer"): + nb.plan_complete_request_budget(**case) + + +def test_complete_request_plan_covers_a_127_model_catalog() -> None: + """The reviewed current-catalog scale fits only when probes and eval are reserved.""" + plan = nb.plan_complete_request_budget( + discovered_model_count=127, + max_eval_models=7, + locked_task_count=10, + ) + + assert plan == { + "catalog_request_count": 1, + "capability_probe_request_count": 127 * 9, + "evaluation_reserve_request_count": 260, + "planned_worker_count": 7, + "total_required_request_count": 1404, + } + + +def test_buyer_facing_request_plan_matches_internal_plan() -> None: + """The stable operator view exposes the same complete-run reservation.""" + assert nb.planned_complete_run_requests(127, 30, 7) == { + "catalog_discovery_requests": 1, + "capability_probe_requests": 127 * 9, + "evaluation_worker_ceiling": 7, + "evaluation_requests": 780, + "requests_after_catalog": 127 * 9 + 780, + "total_requests": 1924, + } + + +def test_one_request_short_fails_after_catalog_before_any_probe(tmp_path: Path) -> None: + """An undersized live-style plan spends discovery only, then fails closed.""" + model_rows = [ + {"id": f"vendor/model-{index:03d}", "owned_by": "vendor"} + for index in range(127) + ] + calls: list[tuple[str, str]] = [] + + def transport( + method: str, + url: str, + _headers: dict[str, str], + _body: bytes | None, + ) -> tuple[int, bytes]: + calls.append((method, urllib.parse.urlparse(url).path)) + if method == "GET": + return 200, json.dumps({"data": model_rows}).encode("utf-8") + raise AssertionError("capability egress must not begin after failed preflight") + + with pytest.raises( + nb.BenchmarkBudgetError, + match="complete benchmark needs 1924 requests but configured cap is 1923", + ): + nb.run_benchmark( + "dry_run", + TASK_MANIFEST_PATH, + None, + str(tmp_path / "insufficient"), + endpoint=FAKE_ENDPOINT, + max_total_requests=1923, + max_eval_models=7, + transport=transport, + ) + + assert calls == [("GET", "/v1/models")] + + +def test_exact_complete_request_boundary_runs_and_records_plan(tmp_path: Path) -> None: + """The exact conservative boundary succeeds and records configured reserves.""" + manifest_path = _write_json( + tmp_path / "boundary_manifest.json", + { + "manifest_version": "boundary.1", + "tasks": [ + { + "task_id": "locked_boundary_task", + "split": "locked", + "prompt": "Name a striped animal.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "zebra"}, + } + ], + }, + ) + + def transport( + method: str, + url: str, + _headers: dict[str, str], + _body: bytes | None, + ) -> tuple[int, bytes]: + path = urllib.parse.urlparse(url).path + if method == "GET": + return 200, json.dumps( + {"data": [{"id": "vendor/model-one", "owned_by": "vendor"}]} + ).encode("utf-8") + return 200, nb._dry_run_success_body(path) + + report = nb.run_benchmark( + "dry_run", + manifest_path, + None, + str(tmp_path / "exact_boundary"), + endpoint=FAKE_ENDPOINT, + max_total_requests=24, + max_eval_models=1, + transport=transport, + ) + + assert report["request_budget"]["max_total_requests"] == 24 + assert report["request_budget"]["planned_total_requests"] == 24 + assert report["request_budget"]["catalog_requests"] == 1 + assert report["request_budget"]["capability_probe_requests"] == 9 + assert report["request_budget"]["evaluation_reserve_requests"] == 14 + assert report["request_budget"]["requests_spent"] <= 24 + + +def test_video_probe_fixture_is_one_decodable_frame_with_stable_hash() -> None: + """A video-capable model receives a real one-frame MP4, not an ftyp-only stub.""" + fixture = nb._tiny_mp4_bytes() + metadata = nb.validate_video_probe_fixture(fixture) + + assert metadata == { + "codec_name": "h264", + "width": 16, + "height": 16, + "frame_count": 1, + } + assert hashlib.sha256(fixture).hexdigest() == nb.VIDEO_PROBE_FIXTURE_SHA256 + assert len(fixture) > 1000 + + +def test_smoke_manifest_cannot_authorize_production_routing(tmp_path: Path) -> None: + """The smoke manifest produces review evidence, not an automatic decision.""" + report = nb.run_benchmark( + "dry_run", + TASK_MANIFEST_PATH, + None, + str(tmp_path), + max_total_requests=600, + max_eval_models=2, + ) + evaluation = report["evaluation"] + + assert evaluation["evidence_status"] == "evidence_review_required" + assert evaluation["decision_use"] == "production_candidate_review" + assert evaluation["minimum_paired_task_count"] == 30 + assert evaluation["required_completion_fraction"] == 0.9 + assert evaluation["routing_recommendation"] is None + assert report["provenance"]["benchmark_parameters"]["policy_total_token_budget"] == ( + nb.DEFAULT_POLICY_TOTAL_TOKEN_BUDGET + ) + assert report["honesty_labels"]["actual_cost_basis"] == ( + "deterministic_dry_run_no_provider_egress" + ) + + +class _BudgetDelegate: + """Minimal provider client used to exercise direct equal-budget behavior.""" + + def __init__(self, answer: str = "ok", usage: object = None) -> None: + """Configure one answer and optional provider usage payload.""" + self.max_output_tokens = 256 + self.answer = answer + self.usage = usage + self.observed_caps: list[int] = [] + + @contextmanager + def request_settings(self, **overrides): + """Apply the scoped output cap used by the real model client.""" + previous = self.max_output_tokens + if overrides.get("max_output_tokens") is not None: + self.max_output_tokens = overrides["max_output_tokens"] + try: + yield + finally: + self.max_output_tokens = previous + + def chat( + self, + _agent, + _messages, + _temperature=None, + _top_p=None, + _effort_profile=None, + ) -> str: + """Record the temporary output cap and return the configured answer.""" + self.observed_caps.append(self.max_output_tokens) + return self.answer + + def take_usage(self): + """Return the configured provider usage payload.""" + return self.usage + + +def _budget_agent(): + """Return one valid mock worker for cell-budget tests.""" + from contextual_orchestrator.orchestrator import ModelAgent + + return ModelAgent( + id="nim_budget_worker", + model="dryrun/chat-basic", + base_url="mock://nim-budget-test", + credential_key=nb.NIM_CREDENTIAL_NAME, + tags=("reasoning", "writing"), + ) + + +def _mp4_box(box_type: bytes, payload: bytes = b"") -> bytes: + """Build one small ISO-BMFF box for malformed-fixture regression tests.""" + import struct + + return struct.pack(">I4s", len(payload) + 8, box_type) + payload + + +def test_default_transport_rejects_invalid_timeout_values() -> None: + """Only finite positive real timeout values can reach socket setup.""" + for value in (False, 0, -1, "5", float("nan"), float("inf")): + with pytest.raises(nb.BenchmarkContractError, match="timeout_seconds"): + nb.build_default_transport(value) + + +def test_equal_budget_client_validates_and_exposes_delegate_cap() -> None: + """Equal budgets are positive integers and preserve the client cap interface.""" + for token_budget in (False, 0, 1.5): + with pytest.raises(ValueError, match="total_token_budget"): + nb.EqualBudgetModelClient(_BudgetDelegate(), token_budget, 5) + for maximum_calls in (False, 0, 1.5): + with pytest.raises(ValueError, match="maximum_calls"): + nb.EqualBudgetModelClient(_BudgetDelegate(), 20, maximum_calls) + + delegate = _BudgetDelegate() + client = nb.EqualBudgetModelClient(delegate, 20, 5) + assert client.max_output_tokens == 256 + client.max_output_tokens = 128 + assert delegate.max_output_tokens == 128 + + client.chat(_budget_agent(), [{"role": "user", "content": "hi"}]) + assert delegate.observed_caps == [11] + + +def test_budgeted_client_uses_the_injected_transport(monkeypatch) -> None: + """Live-style evaluation calls stay on the benchmark transport seam.""" + register_credential(nb.NIM_CREDENTIAL_NAME, "secret-test-key") + calls: list[tuple[str, str, bytes | None]] = [] + + def transport(method, url, headers, body): + calls.append((method, url, body)) + assert headers["authorization"] == "Bearer secret-test-key" + return 200, json.dumps( + { + "choices": [{"message": {"content": "answer"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ).encode() + + monkeypatch.setattr(ModelClient, "_validate_provider", lambda *_args: None) + client = nb._BudgetedModelClient(nb.RequestBudget(1), transport=transport) + agent = _budget_agent() + agent = agent.__class__( + id=agent.id, + model=agent.model, + base_url=FAKE_ENDPOINT, + credential_key=nb.NIM_CREDENTIAL_NAME, + tags=agent.tags, + ) + assert client.chat(agent, [{"role": "user", "content": "hello"}]) == "answer" + assert calls and calls[0][0:2] == ("POST", f"{FAKE_ENDPOINT}/chat/completions") + + +def test_budgeted_client_fallback_and_transport_errors(monkeypatch) -> None: + """The wrapper preserves the base client seam and normalizes provider errors.""" + register_credential(nb.NIM_CREDENTIAL_NAME, "secret-test-key") + agent = _budget_agent() + agent = agent.__class__( + id=agent.id, + model=agent.model, + base_url=FAKE_ENDPOINT, + credential_key=nb.NIM_CREDENTIAL_NAME, + tags=agent.tags, + ) + monkeypatch.setattr(ModelClient, "_validate_provider", lambda *_args: None) + monkeypatch.setattr( + ModelClient, + "_send", + lambda *_args, **_kwargs: "fallback answer", + ) + fallback = nb._BudgetedModelClient(nb.RequestBudget(1)) + assert fallback.chat(agent, [{"role": "user", "content": "hello"}]) == "fallback answer" + + def failing_transport(*_args, **_kwargs): + return 500, b"provider failure" + + failing = nb._BudgetedModelClient(nb.RequestBudget(1), transport=failing_transport) + with pytest.raises(RuntimeError, match="provider rejected the request with HTTP 500"): + failing.chat(agent, [{"role": "user", "content": "hello"}]) + + +@pytest.mark.parametrize("value", [True, "3", float("nan"), float("inf"), -1]) +def test_equal_budget_usage_count_rejects_invalid_values(value: object) -> None: + """Provider token counts must be finite non-negative real numbers, never booleans.""" + assert nb.EqualBudgetModelClient._coerce_usage_count(value) is None + assert nb.EqualBudgetModelClient._coerce_usage_count(3.9) == 3 + + +def test_equal_budget_usage_reconciliation_covers_all_sources() -> None: + """Reported usage replaces estimates only when both counts are usable.""" + no_usage = nb.EqualBudgetModelClient(_BudgetDelegate(usage=None), 100, 5) + assert no_usage.take_usage() is None + + non_mapping = nb.EqualBudgetModelClient(_BudgetDelegate(usage="unknown"), 100, 5) + non_mapping.chat(_budget_agent(), [{"role": "user", "content": "hi"}], 0.0) + estimated_non_mapping = non_mapping.observed_tokens + assert non_mapping.take_usage() == "unknown" + assert non_mapping.observed_tokens == estimated_non_mapping + + invalid_counts = nb.EqualBudgetModelClient( + _BudgetDelegate(usage={"prompt_tokens": True, "completion_tokens": -1}), + 100, + 5, + ) + invalid_counts.chat(_budget_agent(), [{"role": "user", "content": "hi"}], 0.0) + estimated_invalid = invalid_counts.observed_tokens + assert invalid_counts.take_usage() == { + "prompt_tokens": True, + "completion_tokens": -1, + } + assert invalid_counts.observed_tokens == estimated_invalid + + reported = nb.EqualBudgetModelClient( + _BudgetDelegate(usage={"prompt_tokens": 2, "completion_tokens": 3}), + 100, + 5, + ) + reported.chat(_budget_agent(), [{"role": "user", "content": "hi"}], 0.0) + assert reported.take_usage() == {"prompt_tokens": 2, "completion_tokens": 3} + assert reported.observed_tokens == 5 + + +def test_mp4_parser_rejects_every_malformed_box_class() -> None: + """Fixture validation fails closed on truncation, bad bounds, and missing evidence.""" + import struct + + with pytest.raises(nb.BenchmarkContractError, match="truncated box header"): + list(nb._iter_mp4_boxes(b"x")) + with pytest.raises(nb.BenchmarkContractError, match="truncated extended box"): + list(nb._iter_mp4_boxes(struct.pack(">I4s", 1, b"free"))) + + extended = struct.pack(">I4sQ", 1, b"free", 16) + assert list(nb._iter_mp4_boxes(extended)) == [(b"free", 16, 16)] + zero_sized = struct.pack(">I4s", 0, b"free") + b"payload" + assert list(nb._iter_mp4_boxes(zero_sized)) == [ + (b"free", 8, len(zero_sized)) + ] + with pytest.raises(nb.BenchmarkContractError, match="parent bounds"): + list(nb._iter_mp4_boxes(struct.pack(">I4s", 20, b"free"))) + + with pytest.raises(nb.BenchmarkContractError, match="meta box"): + list(nb._walk_mp4_boxes(_mp4_box(b"meta"))) + with pytest.raises(nb.BenchmarkContractError, match="lacks ftyp"): + nb.validate_video_probe_fixture(_mp4_box(b"ftyp")) + + required_top_level = _mp4_box(b"ftyp") + _mp4_box(b"moov") + _mp4_box(b"mdat") + with pytest.raises(nb.BenchmarkContractError, match="one 16x16 one-frame"): + nb.validate_video_probe_fixture(required_top_level) + + truncated_tkhd = ( + _mp4_box(b"ftyp") + + _mp4_box(b"moov", _mp4_box(b"tkhd", b"x")) + + _mp4_box(b"mdat") + ) + with pytest.raises(nb.BenchmarkContractError, match="tkhd box"): + nb.validate_video_probe_fixture(truncated_tkhd) + + truncated_stsz = ( + _mp4_box(b"ftyp") + + _mp4_box(b"moov", _mp4_box(b"stsz", b"short")) + + _mp4_box(b"mdat") + ) + with pytest.raises(nb.BenchmarkContractError, match="stsz box"): + nb.validate_video_probe_fixture(truncated_stsz) + + +def test_video_fixture_checksum_mismatch_fails_closed(monkeypatch: pytest.MonkeyPatch) -> None: + """A changed embedded media payload cannot silently enter provider probes.""" + monkeypatch.setattr(nb, "VIDEO_PROBE_FIXTURE_SHA256", "0" * 64) + with pytest.raises(nb.BenchmarkContractError, match="checksum"): + nb._tiny_mp4_bytes() + + +def test_reviewed_pricing_metadata_rejects_invalid_provenance(tmp_path: Path) -> None: + """Reviewed scenarios require valid dates, HTTPS source, and non-empty review fields.""" + invalid_scenarios = [ + _reviewed_pricing_scenario(source_url="http://pricing.example.test/rates"), + _reviewed_pricing_scenario(reviewed_by=""), + _reviewed_pricing_scenario(rate_basis=""), + _reviewed_pricing_scenario(uncertainty=""), + _reviewed_pricing_scenario(reviewed_at_date=3), + _reviewed_pricing_scenario(reviewed_at_date="not-a-date"), + _reviewed_pricing_scenario( + reviewed_at_date="2026-08-05", valid_until_date="2026-08-04" + ), + _reviewed_pricing_scenario( + usd_per_million_tokens={"": {"input": 1.0, "output": 2.0}} + ), + ] + for index, scenario in enumerate(invalid_scenarios): + path = _write_json(tmp_path / f"invalid_reviewed_{index}.json", scenario) + with pytest.raises(nb.BenchmarkContractError): + nb.load_pricing_scenario(path) + + +def test_live_pricing_rejects_future_review_and_accepts_current_evidence() -> None: + """A live run date must fall within the reviewed pricing validity interval.""" + future = _reviewed_pricing_scenario( + reviewed_at_date="2026-08-06", valid_until_date="2026-09-04" + ) + with pytest.raises(nb.BenchmarkContractError, match="future"): + nb.validate_live_pricing_scenario( + future, + today=__import__("datetime").date(2026, 8, 5), + ) + nb.validate_live_pricing_scenario( + _reviewed_pricing_scenario(), + today=__import__("datetime").date(2026, 8, 5), + ) + + +def test_actual_cost_evidence_validation_and_expiry_paths( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The zero access-cost claim remains complete, official, and time bounded.""" + with pytest.raises(nb.BenchmarkContractError, match="missing actual_cost_evidence"): + nb._validate_actual_cost_evidence({}) + + missing = {"actual_cost_evidence": dict(nb.ACTUAL_COST_EVIDENCE)} + del missing["actual_cost_evidence"]["source_title"] + with pytest.raises(nb.BenchmarkContractError, match="missing fields"): + nb._validate_actual_cost_evidence(missing) + + wrong_cost = {"actual_cost_evidence": dict(nb.ACTUAL_COST_EVIDENCE)} + wrong_cost["actual_cost_evidence"]["actual_cost_usd"] = 1.0 + with pytest.raises(nb.BenchmarkContractError, match="zero-cost"): + nb._validate_actual_cost_evidence(wrong_cost) + + wrong_source = {"actual_cost_evidence": dict(nb.ACTUAL_COST_EVIDENCE)} + wrong_source["actual_cost_evidence"]["source_url"] = "https://example.test" + with pytest.raises(nb.BenchmarkContractError, match="General FAQ"): + nb._validate_actual_cost_evidence(wrong_source) + + invalid_dates = {"actual_cost_evidence": dict(nb.ACTUAL_COST_EVIDENCE)} + invalid_dates["actual_cost_evidence"]["reviewed_at_date"] = "2026-09-05" + with pytest.raises(nb.BenchmarkContractError, match="validity precedes"): + nb._validate_actual_cost_evidence(invalid_dates) + + monkeypatch.setitem(nb.ACTUAL_COST_EVIDENCE, "reviewed_at_date", "2026-08-06") + with pytest.raises(nb.BenchmarkContractError, match="future"): + nb._require_current_actual_cost_evidence( + __import__("datetime").date(2026, 8, 5) + ) + monkeypatch.setitem(nb.ACTUAL_COST_EVIDENCE, "reviewed_at_date", "2026-08-05") + monkeypatch.setitem(nb.ACTUAL_COST_EVIDENCE, "valid_until_date", "2026-08-05") + nb._require_current_actual_cost_evidence(__import__("datetime").date(2026, 8, 5)) + with pytest.raises(nb.BenchmarkContractError, match="expired"): + nb._require_current_actual_cost_evidence( + __import__("datetime").date(2026, 8, 6) + ) + + +def test_sufficient_evidence_is_still_human_review_gated() -> None: + """Meeting sample thresholds changes status but never auto-selects a route.""" + cells = [] + for task_index in range(nb.MINIMUM_PAIRED_TASK_COUNT): + task_id = f"paired_task_{task_index}" + for policy_name in ("route_once", "conduct_bounded"): + cells.append( + { + "policy_name": policy_name, + "task_id": task_id, + "run_outcome": "success", + } + ) + summary = nb._evaluation_evidence_summary( + cells, + nb.MINIMUM_PAIRED_TASK_COUNT, + ) + assert summary["evidence_status"] == "evidence_review_required" + assert summary["decision_use"] == "production_candidate_review" + assert summary["routing_recommendation"] is None + + +def test_live_run_requires_provenance_before_transport(tmp_path: Path) -> None: + """Missing live revision identity fails before credentials or transport are used.""" + with pytest.raises(nb.BenchmarkContractError, match="git-sha"): + nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + None, + str(tmp_path), + transport=_unexpected_transport, + ) diff --git a/tests/test_nim_benchmark_workflow_contract.py b/tests/test_nim_benchmark_workflow_contract.py new file mode 100644 index 000000000..5440f44fd --- /dev/null +++ b/tests/test_nim_benchmark_workflow_contract.py @@ -0,0 +1,98 @@ +"""Static least-privilege contracts for scheduled NIM benchmark automation.""" + +from __future__ import annotations + +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + +def test_dry_run_workflow_never_receives_live_nvidia_secret() -> None: + """The zero-egress dry path must have no NVIDIA credential in its environment.""" + workflow = (REPOSITORY_ROOT / ".github/workflows/nim-benchmark.yml").read_text( + encoding="utf-8" + ) + dry_start = workflow.index("- name: Run dry benchmark") + live_start = workflow.index("- name: Run live benchmark") + dry_block = workflow[dry_start:live_start] + live_block = workflow[live_start:] + + assert "NVIDIA_NIM_API_KEY" not in dry_block + assert live_block.count("NVIDIA_NIM_API_KEY:") == 1 + assert "secrets.NVIDIA_NIM_API_KEY" in live_block + + +def test_dry_run_workflow_honors_optional_pricing_scenario_without_secret() -> None: + """Manual dry runs may validate an explicit scenario without live credentials.""" + workflow = (REPOSITORY_ROOT / ".github/workflows/nim-benchmark.yml").read_text( + encoding="utf-8" + ) + dry_start = workflow.index("- name: Run dry benchmark") + live_start = workflow.index("- name: Run live benchmark") + dry_block = workflow[dry_start:live_start] + + assert "PRICING_SCENARIO: ${{ inputs.pricing_scenario }}" in dry_block + assert 'extra_args+=(--pricing-scenario "$PRICING_SCENARIO")' in dry_block + assert '"${extra_args[@]}"' in dry_block + + +def test_temporary_review_export_job_is_absent_from_mergeable_tests_workflow() -> None: + """Mergeable CI must not retain the one-use exact-head export mechanism.""" + workflow = (REPOSITORY_ROOT / ".github/workflows/ci.yml").read_text( + encoding="utf-8" + ) + assert "export_review_workspace" not in workflow + assert "Recover reviewed transformation source as inert evidence" not in workflow + + +def test_temporary_review_evidence_source_is_absent() -> None: + """Mergeable source must not retain any one-use transformation payload.""" + assert not (REPOSITORY_ROOT / ".review-evidence/nim-source-repair.yml").exists() + assert not ( + REPOSITORY_ROOT / ".github/workflows/export-pr90-workspace.yml" + ).exists() + + +def test_compatibility_monkeypatch_module_is_absent() -> None: + """Security and budget behavior must live directly in the optional benchmark.""" + assert not ( + REPOSITORY_ROOT / "contextual_orchestrator/nim_benchmark_hardening.py" + ).exists() + assert not (REPOSITORY_ROOT / "tests/test_nim_benchmark_hardening.py").exists() + + +def test_tests_workflow_enforces_nim_coverage_docstrings_and_package_smoke() -> None: + """The exact PR head must prove 100% branches, docstrings, and installability.""" + workflow = (REPOSITORY_ROOT / ".github/workflows/ci.yml").read_text( + encoding="utf-8" + ) + assert "nim_benchmark_quality:" in workflow + assert "coverage run --branch" in workflow + assert "--source=contextual_orchestrator.nim_benchmark" in workflow + assert "coverage report" in workflow and "--fail-under=100" in workflow + assert "interrogate -f 100 contextual_orchestrator/nim_benchmark.py" in workflow + assert "pip wheel --no-deps --no-build-isolation" in workflow + assert '--target "$RUNNER_TEMP/nim-wheel-site"' in workflow + assert 'cd "$RUNNER_TEMP"' in workflow + assert 'PYTHONPATH="$RUNNER_TEMP/nim-wheel-site"' in workflow + assert "import contextual_orchestrator.nim_benchmark" in workflow + + +def test_scheduled_live_budget_covers_the_reviewed_current_catalog_scale() -> None: + """Monthly live runs reserve enough calls for full probes plus evaluation.""" + workflow = (REPOSITORY_ROOT / ".github/workflows/nim-benchmark.yml").read_text( + encoding="utf-8" + ) + + assert 'echo "max_requests=2000" >> "$GITHUB_OUTPUT"' in workflow + assert 'echo "max_requests=300" >> "$GITHUB_OUTPUT"' not in workflow + + +def test_monthly_schedule_starts_inside_the_reviewed_evidence_window() -> None: + """The next monthly run must precede the current evidence expiry date.""" + workflow = (REPOSITORY_ROOT / ".github/workflows/nim-benchmark.yml").read_text( + encoding="utf-8" + ) + + assert 'cron: "23 3 1 * *"' in workflow