fix(security): pin provider egress and repair the Atheris lock

.github/workflows/fuzz.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
   schedule:
     # Weekly deeper run (longer per-target budget via FUZZ_SECONDS).
     - cron: "41 4 * * 2"
@@ -26,6 +25,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Set up Python
@@ -50,6 +50,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Set up Python

.github/workflows/provider-catalog-sync.yml

@@ -0,0 +1,146 @@
+name: Provider Catalog Sync
+
+on:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: "17 */6 * * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: provider-catalog-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  contract:
+    name: Offline provider-catalog contracts
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout exact revision
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install hash-locked test dependencies
+        run: |
+          python -m pip install --require-hashes -r requirements-opencode-review-ci.txt
+          python -m pip install --require-hashes -r fuzz/requirements-property.txt
+
+      - name: Run provider-catalog contracts
+        run: |
+          python -m pytest tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py -q
+          python -m compileall -q contextual_orchestrator
+
+  synchronize:
+    name: Seed credentials and refresh durable catalog
+    if: >-
+      github.event_name != 'pull_request' &&
+      github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    environment: production
+    env:
+      CONTEXTUAL_ORCHESTRATOR_KV_BACKEND: postgres
+      CONTEXTUAL_ORCHESTRATOR_KV_DSN: ${{ secrets.CONTEXTUAL_ORCHESTRATOR_KV_DSN }}
+      CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN: ${{ secrets.CONTEXTUAL_ORCHESTRATOR_KV_DSN }}
+      CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE: ${{ secrets.CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE }}
+      NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}
+      NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }}
+      BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }}
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+    steps:
+      - name: Checkout protected default-branch revision
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install hash-locked runtime and database dependencies
+        run: python -m pip install --require-hashes -r requirements.lock
+
+      - name: Validate trusted bootstrap inventory
+        shell: bash
+        run: |
+          set +x
+          required=(
+            CONTEXTUAL_ORCHESTRATOR_KV_DSN
+            CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE
+            NVIDIA_NIM_API_KEY
+            NVIDIA_NIM_API_KEY_SUB
+            BYTEZ_API_KEY
+            OPENROUTER_API_KEY
+            OPENAI_API_KEY
+          )
+          for name in "${required[@]}"; do
+            value="${!name:-}"
+            if [[ -z "$value" ]]; then
+              echo "::error title=Provider catalog bootstrap blocked::Required secret $name is not configured"
+              exit 2
+            fi
+            echo "::add-mask::$value"
+          done
+
+      - name: Seed encrypted credential registry and refresh model catalog
+        shell: bash
+        run: |
+          set +x
+          python -m contextual_orchestrator.provider_catalog \
+            bootstrap-and-sync \
+            --require-all \
+            --agents-output "$RUNNER_TEMP/provider-agents.json" \
+            > "$RUNNER_TEMP/provider-catalog-summary.json"
+
+      - name: Verify secret-free generated agent pool
+        shell: bash
+        run: |
+          python - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          agents_path = Path(os.environ["RUNNER_TEMP"]) / "provider-agents.json"
+          summary_path = Path(os.environ["RUNNER_TEMP"]) / "provider-catalog-summary.json"
+          agents = json.loads(agents_path.read_text(encoding="utf-8"))["agents"]
+          summary = json.loads(summary_path.read_text(encoding="utf-8"))
+          if not agents:
+              raise SystemExit("provider catalog produced no candidate agents")
+          forbidden = {
+              os.environ[name]
+              for name in (
+                  "NVIDIA_NIM_API_KEY",
+                  "NVIDIA_NIM_API_KEY_SUB",
+                  "BYTEZ_API_KEY",
+                  "OPENROUTER_API_KEY",
+                  "OPENAI_API_KEY",
+              )
+          }
+          serialized = json.dumps({"agents": agents, "summary": summary})
+          if any(secret and secret in serialized for secret in forbidden):
+              raise SystemExit("generated provider evidence contains a secret value")
+          print(json.dumps({
+              "candidate_agent_count": len(agents),
+              "candidate_model_count": summary["candidate_model_count"],
+              "measurement_status": summary["measurement_status"],
+          }, sort_keys=True))
+          PY
+
+      - name: Confirm runtime secret-source boundary
+        shell: bash
+        run: |
+          unset NVIDIA_NIM_API_KEY NVIDIA_NIM_API_KEY_SUB BYTEZ_API_KEY OPENROUTER_API_KEY OPENAI_API_KEY
+          echo "Provider credentials are persisted in the encrypted KV registry; runtime resolves names only."

.github/workflows/security.yml

@@ -9,7 +9,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
   schedule:
     - cron: "17 3 * * 1"
   workflow_dispatch:
@@ -34,6 +33,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Initialize CodeQL
@@ -54,6 +54,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Set up Python

.github/workflows/tests.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
 
 permissions:
   contents: read
@@ -15,24 +14,35 @@ concurrency:
 
 jobs:
   pytest:
-    name: Full unit and contract suite
+    name: Full unit and contract suite (Python ${{ matrix.python-version }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.12"]
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Set up Python
         uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
         with:
-          python-version: "3.12"
+          python-version: ${{ matrix.python-version }}
 
       - name: Install test dependencies
-        # Hash-pinned per OpenSSF Scorecard Pinned-Dependencies. Reuses the
-        # property-test lockfile (pytest + hypothesis), which covers the full
-        # suite's requirements: the package itself is stdlib-only.
-        run: python -m pip install --require-hashes -r fuzz/requirements-property.txt
+        # Both inputs are hash-locked. The review-tool lock provides coverage
+        # and interrogate; the property-test lock is installed last so the
+        # repository keeps its existing pytest + Hypothesis test environment.
+        run: |
+          python -m pip install --require-hashes -r requirements-opencode-review-ci.txt
+          python -m pip install --require-hashes -r fuzz/requirements-property.txt
 
       - name: Run full test suite
-        run: python -m pytest -q
+        run: |
+          python -m coverage erase
+          python -m coverage run --branch -m pytest -q
+          python -m coverage report --fail-under=100
+          interrogate --fail-under 100 contextual_orchestrator

CHANGELOG.md

@@ -0,0 +1,48 @@
+# Changelog
+
+All notable changes to Contextual Orchestrator are documented in this file.
+
+The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+- Add a durable, normalized provider catalog for the organization `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, `BYTEZ_API_KEY`, `OPENROUTER_API_KEY`, and `OPENAI_API_KEY` accounts; trusted bootstrap writes values only to the encrypted credential registry, discovers provider models account by account, preserves last-known-good catalogs on isolated failures, generates role-tagged agents, and starts the gateway from enabled database candidates with `--provider-catalog-dsn`.
+- Add a provider-aware runtime client that preserves the hardened OpenAI-compatible transport for OpenAI, OpenRouter, and NVIDIA NIM while using a narrow native Bytez Key/input adapter and failing closed for unsupported Bytez passthrough response shapes.
+- Add a trust-separated Provider Catalog Sync workflow: pull requests run secret-free offline contracts, while protected-main scheduled/manual runs require the complete five-key inventory plus durable KV DSN/passphrase, verify generated evidence contains no secret value, and never downgrade a configured database to process memory.
+
+### Security
+
+- Fail closed with a stable redacted error when an explicitly configured Postgres KV backend cannot be imported, initialized, or seeded, and route `CostRoutingCoordinator(postgres_dsn=...)` through that authoritative factory, preventing a silent downgrade of configuration, routing, price, and credential authority to process-local memory.
+- Restrict the private plain-HTTP provider seam to `localhost` or literal loopback IP addresses, reject URL userinfo before connection, dial directly without ambient proxy lookup, reject all redirect responses, and close failed resources deterministically.
+- Pin each HTTPS provider connection to the exact public addresses approved during validation, preserve the original hostname for TLS verification, bypass environment proxy resolution, and reject redirects to close DNS-rebinding and credential-forwarding SSRF paths.
+- Fail closed at the final pre-socket HTTPS boundary when a provider Bearer credential is missing or empty at dispatch time, so credential revocation after DNS validation cannot degrade into unauthenticated provider network egress.
+- Bound every provider response to 8 MiB of cumulative consumed bytes, including SSE iteration, reject oversized declared lengths before body consumption, fail closed on malformed or conflicting `Content-Length` and ambiguous `Content-Length` plus `Transfer-Encoding`, redact header-inspection failures, and never silently truncate an untrusted response.
+- Accept only the single HTTP/1.1 `chunked` provider `Transfer-Encoding` that the reviewed standard-library transport decodes, and fail closed on unsupported transfer codings or coding chains before application model-output parsing.
+- Require a real provider streaming response to advertise the `text/event-stream` media type before any streamed body line is consumed, accepting media-type parameters but rejecting missing or incompatible types and redacting header-access failures.
+- Reject malformed UTF-8 in accepted provider SSE streams with one stable redacted error, preventing provider-controlled decoder detail from crossing the transport trust boundary while preserving deterministic cleanup.
+- Fail closed when an accepted OpenAI-compatible SSE provider stream contains malformed `data:` JSON or reaches EOF before its terminal `data: [DONE]` marker, preventing truncated model output from being accepted as successful orchestration evidence.
+- Reject malformed UTF-8/JSON, duplicate object names, Python non-finite-number extensions, finite-syntax exponents that overflow Python floats to non-finite values, and non-object top-level values in validated structured provider responses before application parsing; canonicalize valid JSON and strict Batch JSON Lines so later decoder failures cannot retain the original provider document.
+- Integrate DNS-pinned provider dispatch directly into `ModelClient` so package import performs no optional-adapter monkey-patching or order-dependent class mutation.
+- Reject provider hosts that resolve to any non-globally-routable address, including RFC 6598 shared address space, while retaining explicit multicast, private, loopback, link-local, and reserved-address protections.
+- Document narrowly scoped Semgrep suppressions for parameter-bound database queries, the explicit development-only TLS verification opt-out, and provider URLs that pass the egress guard.
+
+### Changed
+
+- Build and inspect normal wheel and sdist artifacts to enforce emitted PEP 639 license fields, project URLs, and packaged license paths.
+- Declare the MIT SPDX license, packaged license file, authoritative project URLs, and current provider-neutral orchestration-control-plane description in distribution metadata, and pin the PEP 639-capable setuptools build backend.
+- Pin Atheris by Python interpreter so the Python 3.11 fuzz job and the newer central coverage-evidence image both install a published, hash-locked wheel.
+- Run repository Tests, Fuzz, and Security workflows for stacked pull requests targeting any branch, bind every checkout to the literal contributor-head SHA, and keep checkout credentials non-persistent so local evidence cannot silently become absent or synthetic-merge-only evidence.
+
+### Documentation
+
+- Add durable provider-catalog design, implementation plan, operator guide, and APA 7 doctoring covering credential/catalog separation, normalized data, account-isolated refresh, route/conduct pool construction, native Bytez handling, trusted Actions bootstrap, rotation, incident response, evidence interpretation, and rollback.
+- Add APA 7 doctoring for Python environment-marker semantics, Atheris artifact availability and hashes, and the supported-platform uncertainty boundary.
+- Add provider-response resource-bound doctoring covering the 8 MiB fail-closed limit, HTTP framing preflight, `text/event-stream` media-type enforcement, bounded SSE reads, OpenAI-compatible `[DONE]` completion evidence, malformed-event and premature-EOF handling, batch-output partitioning, incident handling, and operational rollback.
+- Add provider-stream UTF-8 doctoring grounding strict SSE/JSON decoding and redacted malformed-input handling in the WHATWG HTML Standard and RFC 8259, with verification, failure, rollback, and authority boundaries.
+- Add provider-JSON trust-boundary doctoring grounding strict UTF-8 object decoding, duplicate-name and non-finite-number rejection, finite-runtime numeric enforcement for extreme exponents, Batch JSONL validation, redacted parser failures, request-path authority, operator recovery, and rollback in RFC 8259, current Python documentation, and the OpenAI Batch API contract.
+- Add provider transfer-coding doctoring that distinguishes full RFC 9112 protocol validity from the product's intentionally narrower decoded `chunked` subset, with fail-closed compatibility and rollback guidance.
+- Add provider-credential revocation doctoring covering the dispatch-time race, final pre-socket Bearer guard, operator recovery, compatibility boundary, rollback invariant, and current IETF HTTP/OAuth references.
+- Add pull-request exact-head workflow doctoring covering stacked-base support, contributor-head identity, untrusted-code execution, merge-tree separation, cancellation handling, and rollback.
+- Record the CI trust boundary between generic coverage and native fuzz execution, including the evidence-preserving retry rule for branch-referenced reusable workflows.

contextual_orchestrator/__init__.py

@@ -1,4 +1,8 @@
-"""Public package exports for the contextual orchestration runtime."""
+"""Public package exports for the contextual orchestration runtime.
+
+Importing this module is intentionally side-effect free: provider transports and
+optional adapters are configured explicitly by their owning runtime components.
+"""
 
 from .batch_routing import (
     BatchJob,
@@ -36,8 +40,28 @@
 )
 from .cost_router import CostRoutingCoordinator
 from .credentials import NotConfigured, get_credential, register_credential
-from .kv_config import InMemoryConfigStore, get_config_store
+from .kv_config import (
+    ConfigBackendUnavailableError,
+    InMemoryConfigStore,
+    get_config_store,
+)
 from .orchestrator import ModelAgent, TaskOrchestrator, WorkflowStep, load_agents
+from .provider_catalog import (
+    DEFAULT_PROVIDER_ACCOUNTS,
+    CatalogHttpError,
+    CatalogModelRecord,
+    DiscoveredModel,
+    InMemoryProviderCatalogStore,
+    PostgresProviderCatalogStore,
+    ProviderAccount,
+    ProviderAwareModelClient,
+    ProviderCatalogHttpClient,
+    ProviderCatalogService,
+    ProviderCatalogUnavailable,
+    bootstrap_provider_credentials,
+    build_catalog_orchestrator,
+    normalize_models_document,
+)
 from .token_counting import HeuristicTokenCounter, build_token_counter
 
 __all__ = [
@@ -48,6 +72,21 @@
     "get_credential",
     "register_credential",
     "NotConfigured",
+    # durable provider catalog
+    "DEFAULT_PROVIDER_ACCOUNTS",
+    "ProviderAccount",
+    "DiscoveredModel",
+    "CatalogModelRecord",
+    "CatalogHttpError",
+    "ProviderCatalogUnavailable",
+    "InMemoryProviderCatalogStore",
+    "PostgresProviderCatalogStore",
+    "ProviderCatalogHttpClient",
+    "ProviderAwareModelClient",
+    "ProviderCatalogService",
+    "bootstrap_provider_credentials",
+    "normalize_models_document",
+    "build_catalog_orchestrator",
     # cost review
     "ATTRIBUTION_DIMENSIONS",
     "AttributionDimensions",
@@ -66,6 +105,7 @@
     "dimension_catalog",
     # config / tokens
     "InMemoryConfigStore",
+    "ConfigBackendUnavailableError",
     "get_config_store",
     "HeuristicTokenCounter",
     "build_token_counter",