diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index cfa62adad..3922176de 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: [main] schedule: # Weekly deeper run (longer per-target budget via FUZZ_SECONDS). - cron: "41 4 * * 2" @@ -26,6 +25,7 @@ jobs: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Set up Python @@ -50,6 +50,7 @@ jobs: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Set up Python diff --git a/.github/workflows/provider-catalog-sync.yml b/.github/workflows/provider-catalog-sync.yml new file mode 100644 index 000000000..d4357b4ca --- /dev/null +++ b/.github/workflows/provider-catalog-sync.yml @@ -0,0 +1,146 @@ +name: Provider Catalog Sync + +on: + pull_request: + workflow_dispatch: + schedule: + - cron: "17 */6 * * *" + +permissions: + contents: read + +concurrency: + group: provider-catalog-${{ github.ref }} + cancel-in-progress: false + +jobs: + contract: + name: Offline provider-catalog contracts + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout exact revision + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install hash-locked test dependencies + run: | + python -m pip install --require-hashes -r requirements-opencode-review-ci.txt + python -m pip install --require-hashes -r fuzz/requirements-property.txt + + - name: Run provider-catalog contracts + run: | + python -m pytest tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py -q + python -m compileall -q contextual_orchestrator + + synchronize: + name: Seed credentials and refresh durable catalog + if: >- + github.event_name != 'pull_request' && + github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: production + env: + CONTEXTUAL_ORCHESTRATOR_KV_BACKEND: postgres + CONTEXTUAL_ORCHESTRATOR_KV_DSN: ${{ secrets.CONTEXTUAL_ORCHESTRATOR_KV_DSN }} + CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN: ${{ secrets.CONTEXTUAL_ORCHESTRATOR_KV_DSN }} + CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE: ${{ secrets.CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE }} + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} + BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + steps: + - name: Checkout protected default-branch revision + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install hash-locked runtime and database dependencies + run: python -m pip install --require-hashes -r requirements.lock + + - name: Validate trusted bootstrap inventory + shell: bash + run: | + set +x + required=( + CONTEXTUAL_ORCHESTRATOR_KV_DSN + CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE + NVIDIA_NIM_API_KEY + NVIDIA_NIM_API_KEY_SUB + BYTEZ_API_KEY + OPENROUTER_API_KEY + OPENAI_API_KEY + ) + for name in "${required[@]}"; do + value="${!name:-}" + if [[ -z "$value" ]]; then + echo "::error title=Provider catalog bootstrap blocked::Required secret $name is not configured" + exit 2 + fi + echo "::add-mask::$value" + done + + - name: Seed encrypted credential registry and refresh model catalog + shell: bash + run: | + set +x + python -m contextual_orchestrator.provider_catalog \ + bootstrap-and-sync \ + --require-all \ + --agents-output "$RUNNER_TEMP/provider-agents.json" \ + > "$RUNNER_TEMP/provider-catalog-summary.json" + + - name: Verify secret-free generated agent pool + shell: bash + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + agents_path = Path(os.environ["RUNNER_TEMP"]) / "provider-agents.json" + summary_path = Path(os.environ["RUNNER_TEMP"]) / "provider-catalog-summary.json" + agents = json.loads(agents_path.read_text(encoding="utf-8"))["agents"] + summary = json.loads(summary_path.read_text(encoding="utf-8")) + if not agents: + raise SystemExit("provider catalog produced no candidate agents") + forbidden = { + os.environ[name] + for name in ( + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "BYTEZ_API_KEY", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", + ) + } + serialized = json.dumps({"agents": agents, "summary": summary}) + if any(secret and secret in serialized for secret in forbidden): + raise SystemExit("generated provider evidence contains a secret value") + print(json.dumps({ + "candidate_agent_count": len(agents), + "candidate_model_count": summary["candidate_model_count"], + "measurement_status": summary["measurement_status"], + }, sort_keys=True)) + PY + + - name: Confirm runtime secret-source boundary + shell: bash + run: | + unset NVIDIA_NIM_API_KEY NVIDIA_NIM_API_KEY_SUB BYTEZ_API_KEY OPENROUTER_API_KEY OPENAI_API_KEY + echo "Provider credentials are persisted in the encrypted KV registry; runtime resolves names only." diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 631503e18..fa76a22e1 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -9,7 +9,6 @@ on: push: branches: [main] pull_request: - branches: [main] schedule: - cron: "17 3 * * 1" workflow_dispatch: @@ -34,6 +33,7 @@ jobs: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Initialize CodeQL @@ -54,6 +54,7 @@ jobs: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Set up Python diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 02605dd71..062c6f1d9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: [main] permissions: contents: read @@ -15,24 +14,35 @@ concurrency: jobs: pytest: - name: Full unit and contract suite + name: Full unit and contract suite (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.12"] steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 with: - python-version: "3.12" + python-version: ${{ matrix.python-version }} - name: Install test dependencies - # Hash-pinned per OpenSSF Scorecard Pinned-Dependencies. Reuses the - # property-test lockfile (pytest + hypothesis), which covers the full - # suite's requirements: the package itself is stdlib-only. - run: python -m pip install --require-hashes -r fuzz/requirements-property.txt + # Both inputs are hash-locked. The review-tool lock provides coverage + # and interrogate; the property-test lock is installed last so the + # repository keeps its existing pytest + Hypothesis test environment. + run: | + python -m pip install --require-hashes -r requirements-opencode-review-ci.txt + python -m pip install --require-hashes -r fuzz/requirements-property.txt - name: Run full test suite - run: python -m pytest -q + run: | + python -m coverage erase + python -m coverage run --branch -m pytest -q + python -m coverage report --fail-under=100 + interrogate --fail-under 100 contextual_orchestrator diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..ba9c79da5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,48 @@ +# Changelog + +All notable changes to Contextual Orchestrator are documented in this file. + +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- Add a durable, normalized provider catalog for the organization `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, `BYTEZ_API_KEY`, `OPENROUTER_API_KEY`, and `OPENAI_API_KEY` accounts; trusted bootstrap writes values only to the encrypted credential registry, discovers provider models account by account, preserves last-known-good catalogs on isolated failures, generates role-tagged agents, and starts the gateway from enabled database candidates with `--provider-catalog-dsn`. +- Add a provider-aware runtime client that preserves the hardened OpenAI-compatible transport for OpenAI, OpenRouter, and NVIDIA NIM while using a narrow native Bytez Key/input adapter and failing closed for unsupported Bytez passthrough response shapes. +- Add a trust-separated Provider Catalog Sync workflow: pull requests run secret-free offline contracts, while protected-main scheduled/manual runs require the complete five-key inventory plus durable KV DSN/passphrase, verify generated evidence contains no secret value, and never downgrade a configured database to process memory. + +### Security + +- Fail closed with a stable redacted error when an explicitly configured Postgres KV backend cannot be imported, initialized, or seeded, and route `CostRoutingCoordinator(postgres_dsn=...)` through that authoritative factory, preventing a silent downgrade of configuration, routing, price, and credential authority to process-local memory. +- Restrict the private plain-HTTP provider seam to `localhost` or literal loopback IP addresses, reject URL userinfo before connection, dial directly without ambient proxy lookup, reject all redirect responses, and close failed resources deterministically. +- Pin each HTTPS provider connection to the exact public addresses approved during validation, preserve the original hostname for TLS verification, bypass environment proxy resolution, and reject redirects to close DNS-rebinding and credential-forwarding SSRF paths. +- Fail closed at the final pre-socket HTTPS boundary when a provider Bearer credential is missing or empty at dispatch time, so credential revocation after DNS validation cannot degrade into unauthenticated provider network egress. +- Bound every provider response to 8 MiB of cumulative consumed bytes, including SSE iteration, reject oversized declared lengths before body consumption, fail closed on malformed or conflicting `Content-Length` and ambiguous `Content-Length` plus `Transfer-Encoding`, redact header-inspection failures, and never silently truncate an untrusted response. +- Accept only the single HTTP/1.1 `chunked` provider `Transfer-Encoding` that the reviewed standard-library transport decodes, and fail closed on unsupported transfer codings or coding chains before application model-output parsing. +- Require a real provider streaming response to advertise the `text/event-stream` media type before any streamed body line is consumed, accepting media-type parameters but rejecting missing or incompatible types and redacting header-access failures. +- Reject malformed UTF-8 in accepted provider SSE streams with one stable redacted error, preventing provider-controlled decoder detail from crossing the transport trust boundary while preserving deterministic cleanup. +- Fail closed when an accepted OpenAI-compatible SSE provider stream contains malformed `data:` JSON or reaches EOF before its terminal `data: [DONE]` marker, preventing truncated model output from being accepted as successful orchestration evidence. +- Reject malformed UTF-8/JSON, duplicate object names, Python non-finite-number extensions, finite-syntax exponents that overflow Python floats to non-finite values, and non-object top-level values in validated structured provider responses before application parsing; canonicalize valid JSON and strict Batch JSON Lines so later decoder failures cannot retain the original provider document. +- Integrate DNS-pinned provider dispatch directly into `ModelClient` so package import performs no optional-adapter monkey-patching or order-dependent class mutation. +- Reject provider hosts that resolve to any non-globally-routable address, including RFC 6598 shared address space, while retaining explicit multicast, private, loopback, link-local, and reserved-address protections. +- Document narrowly scoped Semgrep suppressions for parameter-bound database queries, the explicit development-only TLS verification opt-out, and provider URLs that pass the egress guard. + +### Changed + +- Build and inspect normal wheel and sdist artifacts to enforce emitted PEP 639 license fields, project URLs, and packaged license paths. +- Declare the MIT SPDX license, packaged license file, authoritative project URLs, and current provider-neutral orchestration-control-plane description in distribution metadata, and pin the PEP 639-capable setuptools build backend. +- Pin Atheris by Python interpreter so the Python 3.11 fuzz job and the newer central coverage-evidence image both install a published, hash-locked wheel. +- Run repository Tests, Fuzz, and Security workflows for stacked pull requests targeting any branch, bind every checkout to the literal contributor-head SHA, and keep checkout credentials non-persistent so local evidence cannot silently become absent or synthetic-merge-only evidence. + +### Documentation + +- Add durable provider-catalog design, implementation plan, operator guide, and APA 7 doctoring covering credential/catalog separation, normalized data, account-isolated refresh, route/conduct pool construction, native Bytez handling, trusted Actions bootstrap, rotation, incident response, evidence interpretation, and rollback. +- Add APA 7 doctoring for Python environment-marker semantics, Atheris artifact availability and hashes, and the supported-platform uncertainty boundary. +- Add provider-response resource-bound doctoring covering the 8 MiB fail-closed limit, HTTP framing preflight, `text/event-stream` media-type enforcement, bounded SSE reads, OpenAI-compatible `[DONE]` completion evidence, malformed-event and premature-EOF handling, batch-output partitioning, incident handling, and operational rollback. +- Add provider-stream UTF-8 doctoring grounding strict SSE/JSON decoding and redacted malformed-input handling in the WHATWG HTML Standard and RFC 8259, with verification, failure, rollback, and authority boundaries. +- Add provider-JSON trust-boundary doctoring grounding strict UTF-8 object decoding, duplicate-name and non-finite-number rejection, finite-runtime numeric enforcement for extreme exponents, Batch JSONL validation, redacted parser failures, request-path authority, operator recovery, and rollback in RFC 8259, current Python documentation, and the OpenAI Batch API contract. +- Add provider transfer-coding doctoring that distinguishes full RFC 9112 protocol validity from the product's intentionally narrower decoded `chunked` subset, with fail-closed compatibility and rollback guidance. +- Add provider-credential revocation doctoring covering the dispatch-time race, final pre-socket Bearer guard, operator recovery, compatibility boundary, rollback invariant, and current IETF HTTP/OAuth references. +- Add pull-request exact-head workflow doctoring covering stacked-base support, contributor-head identity, untrusted-code execution, merge-tree separation, cancellation handling, and rollback. +- Record the CI trust boundary between generic coverage and native fuzz execution, including the evidence-preserving retry rule for branch-referenced reusable workflows. diff --git a/contextual_orchestrator/__init__.py b/contextual_orchestrator/__init__.py index 70dbd71c6..f22987e36 100644 --- a/contextual_orchestrator/__init__.py +++ b/contextual_orchestrator/__init__.py @@ -1,4 +1,8 @@ -"""Public package exports for the contextual orchestration runtime.""" +"""Public package exports for the contextual orchestration runtime. + +Importing this module is intentionally side-effect free: provider transports and +optional adapters are configured explicitly by their owning runtime components. +""" from .batch_routing import ( BatchJob, @@ -36,8 +40,28 @@ ) from .cost_router import CostRoutingCoordinator from .credentials import NotConfigured, get_credential, register_credential -from .kv_config import InMemoryConfigStore, get_config_store +from .kv_config import ( + ConfigBackendUnavailableError, + InMemoryConfigStore, + get_config_store, +) from .orchestrator import ModelAgent, TaskOrchestrator, WorkflowStep, load_agents +from .provider_catalog import ( + DEFAULT_PROVIDER_ACCOUNTS, + CatalogHttpError, + CatalogModelRecord, + DiscoveredModel, + InMemoryProviderCatalogStore, + PostgresProviderCatalogStore, + ProviderAccount, + ProviderAwareModelClient, + ProviderCatalogHttpClient, + ProviderCatalogService, + ProviderCatalogUnavailable, + bootstrap_provider_credentials, + build_catalog_orchestrator, + normalize_models_document, +) from .token_counting import HeuristicTokenCounter, build_token_counter __all__ = [ @@ -48,6 +72,21 @@ "get_credential", "register_credential", "NotConfigured", + # durable provider catalog + "DEFAULT_PROVIDER_ACCOUNTS", + "ProviderAccount", + "DiscoveredModel", + "CatalogModelRecord", + "CatalogHttpError", + "ProviderCatalogUnavailable", + "InMemoryProviderCatalogStore", + "PostgresProviderCatalogStore", + "ProviderCatalogHttpClient", + "ProviderAwareModelClient", + "ProviderCatalogService", + "bootstrap_provider_credentials", + "normalize_models_document", + "build_catalog_orchestrator", # cost review "ATTRIBUTION_DIMENSIONS", "AttributionDimensions", @@ -66,6 +105,7 @@ "dimension_catalog", # config / tokens "InMemoryConfigStore", + "ConfigBackendUnavailableError", "get_config_store", "HeuristicTokenCounter", "build_token_counter", diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 5f68c3b74..7f11f4673 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -9,6 +9,12 @@ from .credentials import register_credential from .orchestrator import ModelClient, TaskOrchestrator, load_agents +from .provider_catalog import ( + PostgresProviderCatalogStore, + ProviderAwareModelClient, + ProviderCatalogService, + ProviderCatalogUnavailable, +) from .server import SecurityConfig, serve @@ -55,6 +61,22 @@ def _register_credential_command(argv: list[str]) -> None: print(json.dumps({"registered": args.name, "backend": "kv"}, ensure_ascii=False)) +def _runtime_agents(parser: argparse.ArgumentParser, args: argparse.Namespace): + """Load either the durable discovered pool or the explicit seed-file pool.""" + if not args.provider_catalog_dsn: + return load_agents(args.agents) + try: + store = PostgresProviderCatalogStore(args.provider_catalog_dsn) + agents = ProviderCatalogService(store=store).candidate_agents() + except ProviderCatalogUnavailable as exc: + parser.error(str(exc)) + if not agents: + parser.error( + "provider catalog contains no enabled candidates; run the trusted provider-catalog sync first" + ) + return agents + + def main() -> None: """Parse CLI options and run bootstrap, prompt completion, or the HTTP server.""" if len(sys.argv) > 1 and sys.argv[1] == "register-credential": @@ -64,6 +86,14 @@ def main() -> None: parser = argparse.ArgumentParser(description="Route or conduct chat requests across model agents.") parser.add_argument("prompt", nargs="?", help="User prompt for CLI mode.") parser.add_argument("--agents", default="examples/agents.mock.json", help="Agent config JSON.") + parser.add_argument( + "--provider-catalog-dsn", + default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN") or None, + help=( + "Optional PostgreSQL provider-catalog DSN. When set, discovered enabled models " + "replace the seed agent file and provider credentials resolve from the KV registry." + ), + ) parser.add_argument("--state-db", default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_STATE_DB", "") or None, help="Optional sqlite path to persist runs/audit/analytics across restarts (default: in-memory).") parser.add_argument("--mode", choices=["auto", "route", "conduct"], default="auto") @@ -94,9 +124,19 @@ def main() -> None: help="Measure orchestration vs a single-worker baseline on these prompts and print the report.") args = parser.parse_args() - client = ModelClient(ca_bundle=args.provider_ca_bundle, verify_tls=not args.insecure_skip_tls_verify) + agents = _runtime_agents(parser, args) + if args.provider_catalog_dsn: + client = ProviderAwareModelClient( + ca_bundle=args.provider_ca_bundle, + verify_tls=not args.insecure_skip_tls_verify, + ) + else: + client = ModelClient( + ca_bundle=args.provider_ca_bundle, + verify_tls=not args.insecure_skip_tls_verify, + ) orchestrator = TaskOrchestrator( - load_agents(args.agents), + agents, client=client, state_db=args.state_db, agents_db=args.agents_db, diff --git a/contextual_orchestrator/cost_ledger.py b/contextual_orchestrator/cost_ledger.py index d3943c5be..0ccc8f229 100644 --- a/contextual_orchestrator/cost_ledger.py +++ b/contextual_orchestrator/cost_ledger.py @@ -328,6 +328,7 @@ class NoopUsageTelemetrySink: """Default sink for callers that do not wire telemetry yet.""" def emit_usage(self, event: UsageTelemetryEvent) -> None: + """Ignore one prompt-safe usage event when export is not configured.""" return None @@ -340,12 +341,14 @@ def __init__(self, max_events: int = 512) -> None: self._lock = threading.Lock() def emit_usage(self, event: UsageTelemetryEvent) -> None: + """Store one usage event and evict the oldest events above the limit.""" with self._lock: self._events.append(event) if len(self._events) > self._max_events: del self._events[: len(self._events) - self._max_events] def events(self) -> List[UsageTelemetryEvent]: + """Return a stable copy of the currently retained usage events.""" with self._lock: return list(self._events) @@ -361,6 +364,7 @@ class UsageTelemetryHealth: last_error_type: Optional[str] = None def as_dict(self) -> Dict[str, Any]: + """Return counters as a prompt-safe dictionary for operator responses.""" return { "records_accepted": self.records_accepted, "records_stored": self.records_stored, @@ -440,6 +444,7 @@ def flush(self, timeout: Optional[float] = None) -> bool: return True def telemetry_health(self) -> Dict[str, Any]: + """Return a thread-safe snapshot of background ledger export health.""" with self._lock: return self._health.as_dict() @@ -583,8 +588,8 @@ def _seed_dimension_catalog(self) -> None: ph = self._placeholder() cur = self._conn.cursor() for order, (name, label, _column) in enumerate(ATTRIBUTION_DIMENSION_CATALOG): - cur.execute( - f"SELECT 1 FROM cost_attribution_dimensions WHERE dimension_name = {ph}", # nosec B608 - ph is a DB-API placeholder. + cur.execute( # nosemgrep: python.sqlalchemy.security.sqlalchemy-execute-raw-query.sqlalchemy-execute-raw-query + f"SELECT 1 FROM cost_attribution_dimensions WHERE dimension_name = {ph}", # nosec B608 - ph is a DB-API placeholder; the value is bound, not interpolated. (name,), ) if cur.fetchone() is None: @@ -602,8 +607,8 @@ def append(self, record: UsageRecord) -> None: placeholders = ", ".join(ph for _ in _USAGE_COLUMNS) columns = ", ".join(_USAGE_COLUMNS) cur = self._conn.cursor() - cur.execute( - f"INSERT INTO llm_usage_records ({columns}) VALUES ({placeholders})", # nosec B608 - columns are fixed _USAGE_COLUMNS. + cur.execute( # nosemgrep: python.sqlalchemy.security.sqlalchemy-execute-raw-query.sqlalchemy-execute-raw-query + f"INSERT INTO llm_usage_records ({columns}) VALUES ({placeholders})", # nosec B608 - columns are the fixed _USAGE_COLUMNS constant; values are bound. tuple(row.get(column) for column in _USAGE_COLUMNS), ) self._conn.commit() @@ -622,7 +627,7 @@ def query(self, start: Optional[int] = None, end: Optional[int] = None) -> List[ where = f" WHERE {' AND '.join(clauses)}" if clauses else "" columns = ", ".join(_USAGE_COLUMNS) cur = self._conn.cursor() - cur.execute(f"SELECT {columns} FROM llm_usage_records{where}", tuple(params)) # nosec B608 - columns and clauses are fixed. + cur.execute(f"SELECT {columns} FROM llm_usage_records{where}", tuple(params)) # nosec B608 - columns/clauses are fixed templates, values bound. nosemgrep: python.sqlalchemy.security.sqlalchemy-execute-raw-query.sqlalchemy-execute-raw-query return [dict(zip(_USAGE_COLUMNS, values)) for values in cur.fetchall()] @@ -653,7 +658,7 @@ def __init__( ) -> None: self.price_book = price_book self.telemetry_sink = telemetry_sink or NoopUsageTelemetrySink() - base_store = store or InMemoryLedgerStore() + base_store = store if store is not None else InMemoryLedgerStore() should_wrap = bool(non_blocking_store) if should_wrap: self.store: LedgerStore = NonBlockingLedgerStore( diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index bfbe159db..30df2ae0a 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -35,7 +35,7 @@ RoutingPolicy, ) from .cost_ledger import CostLedger, PriceBook -from .kv_config import InMemoryConfigStore +from .kv_config import get_config_store from .token_counting import HeuristicTokenCounter, build_token_counter _EMBEDDING_CONFIG_CATEGORY = "routing" @@ -45,7 +45,12 @@ class CostRoutingCoordinator: - """Wire routing + cost accounting around a ``TaskOrchestrator``.""" + """Wire routing + cost accounting around a ``TaskOrchestrator``. + + An injected config store takes precedence. Otherwise ``postgres_dsn`` + selects the fail-closed durable config factory; omitting both selects the + explicit standalone in-memory default. + """ def __init__( self, @@ -61,7 +66,11 @@ def __init__( postgres_dsn: Optional[str] = None, ) -> None: self.orchestrator = orchestrator - self.config = config_store or InMemoryConfigStore() + self.config = ( + config_store + if config_store is not None + else get_config_store(postgres_dsn) + ) self.price_book = price_book or PriceBook(self.config) self.ledger = ledger or CostLedger(self.price_book) self.token_counter = token_counter or ( @@ -420,15 +429,14 @@ def _force_token_safe_chunks( current = unit else: current = candidate - if current: - chunks.extend( - self._force_token_safe_chunks( - current, - model=model, - max_tokens=max_tokens, - max_chars=max_chars, - ) + chunks.extend( + self._force_token_safe_chunks( + current, + model=model, + max_tokens=max_tokens, + max_chars=max_chars, ) + ) if len(chunks) > 1 or (chunks and chunks[0][0] != text): return chunks @@ -632,8 +640,6 @@ def _weighted_average_embedding(parts: List[tuple[List[float], int]]) -> List[fl return [] dimension = max(len(vector) for vector in vectors) total_weight = sum(max(1, int(weight)) for _vector, weight in parts) - if total_weight <= 0: - total_weight = len(parts) reduced: List[float] = [] for offset in range(dimension): weighted_sum = 0.0 diff --git a/contextual_orchestrator/kv_config.py b/contextual_orchestrator/kv_config.py index fbb7278ae..85bfebb33 100644 --- a/contextual_orchestrator/kv_config.py +++ b/contextual_orchestrator/kv_config.py @@ -4,13 +4,15 @@ batch-backend endpoints, credentials — is read from a KV store, **never** from ``os.getenv`` at runtime. Two backends are provided: -* :class:`InMemoryConfigStore` — the always-available, dependency-free default - used for standalone runs, tests, and the mock/local path. +* :class:`InMemoryConfigStore` — the dependency-free default used only when + callers do not configure a durable backend, including tests and mock/local + paths. * A thin adapter over an installed ``pg_llm_batch.PostgresConfigStore`` / ``pg_llm_batch.SecretStore`` when a Postgres DSN is supplied via :func:`get_config_store`. The DSN itself is the only bootstrap transport; it is passed in explicitly by the caller, not resolved from the environment - here. + here. A configured Postgres backend fails closed instead of silently + downgrading configuration and credential authority to process-local memory. The ``get(category, key, default)`` / ``set(category, key, value)`` shape is deliberately identical to ``pg_llm_batch.PostgresConfigStore`` so the two are @@ -34,6 +36,10 @@ def set(self, category: str, key: str, value: Any) -> None: ... +class ConfigBackendUnavailableError(RuntimeError): + """Raised when a configured durable KV backend cannot be initialized.""" + + class InMemoryConfigStore: """Dependency-free KV config store backed by a nested dict. @@ -129,12 +135,12 @@ def get_config_store( With no DSN, an :class:`InMemoryConfigStore` is returned (the standalone / test default). With a DSN, the ``pg_llm_batch`` Postgres-backed stores are - used when ``pg_llm_batch`` is importable; otherwise the call degrades to the - in-memory store so the orchestrator never hard-depends on Postgres. + authoritative. Import, construction, or seed failures raise a sanitized + :class:`ConfigBackendUnavailableError`; they never downgrade to memory. """ if not postgres_dsn: return InMemoryConfigStore(seed=seed) - try: # pragma: no cover - exercised only with pg_llm_batch + Postgres present + try: from pg_llm_batch import PostgresConfigStore, SecretStore # type: ignore config_store = PostgresConfigStore(postgres_dsn) @@ -145,5 +151,7 @@ def get_config_store( for key, value in entries.items(): adapter.set(category, key, value) return adapter - except Exception: # pragma: no cover - fall back when deps/DB unavailable - return InMemoryConfigStore(seed=seed) + except Exception: + raise ConfigBackendUnavailableError( + "Postgres config backend is unavailable" + ) from None diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 0097b722e..e5bc5df2c 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -8,6 +8,7 @@ from dataclasses import dataclass, replace from functools import wraps import hashlib +import http.client import ipaddress import json import os @@ -27,6 +28,12 @@ from .conventions import require_object_name from .credentials import NotConfigured, get_credential +from .provider_transport import ( + _PinnedHTTPSConnection, + _ProviderHTTPResponse, + _parse_provider_json_object_text, + _validated_public_addresses, +) ChatMessage = dict[str, str] @@ -62,6 +69,22 @@ def estimate_tokens(text: str) -> int: DEFAULT_COMMERCIAL_TARGET_VALUE_KRW = 2_000_000_000 +def _classify_commercial_status( + blocked_count: int, + warning_count: int, + *, + blocked_status: str, + warning_status: str, + ready_status: str, +) -> str: + """Return the public status for blocker, warning, or fully-ready evidence.""" + if blocked_count: + return blocked_status + if warning_count: + return warning_status + return ready_status + + @dataclass(frozen=True) class ModelAgent: """Configuration for one model-backed worker in the agent pool.""" @@ -190,6 +213,19 @@ def as_dict(self) -> dict[str, Any]: TRANSIENT_HTTP_STATUS = frozenset({408, 409, 425, 429, 500, 502, 503, 504}) +def _literal_loopback_host(hostname: str | None) -> bool: + """Return whether a host is localhost or a literal loopback IP address.""" + if hostname is None: + return False + normalized = hostname.rstrip(".").lower() + if normalized == "localhost": + return True + try: + return ipaddress.ip_address(normalized).is_loopback + except ValueError: + return False + + def is_transient_error(exc: BaseException) -> bool: """Return True when a provider call failure is worth retrying with backoff.""" if isinstance(exc, urllib.error.HTTPError): @@ -213,6 +249,8 @@ def __init__( ca_bundle: str | None = None, verify_tls: bool = True, ) -> None: + if max_retries < 0: + raise ValueError("max_retries must be at least zero") self.timeout = timeout self.max_output_tokens = max_output_tokens self.max_retries = max_retries @@ -226,11 +264,14 @@ def __init__( # ca_bundle points at a custom CA (corporate gateways); verify_tls=False is an # explicit dev-only opt-out (insecure) for self-signed endpoints. self._ssl_context = self._build_ssl_context(ca_bundle, verify_tls) + # Explicit test seams; production uses direct loopback HTTP and DNS-pinned TLS. + self._http_connection_class = http.client.HTTPConnection + self._https_connection_class = _PinnedHTTPSConnection @staticmethod def _build_ssl_context(ca_bundle: str | None, verify_tls: bool) -> ssl.SSLContext: if not verify_tls: - return ssl._create_unverified_context() # nosec B323 - explicit dev-only provider TLS opt-out. + return ssl._create_unverified_context() # nosec B323 - explicit dev-only opt-out; default verify_tls=True uses ssl.create_default_context(). nosemgrep: python.lang.security.unverified-ssl-context.unverified-ssl-context if ca_bundle: if not os.path.isfile(ca_bundle): raise ValueError(f"provider CA bundle does not exist: {ca_bundle}") @@ -270,16 +311,15 @@ def chat(self, agent: ModelAgent, messages: list[ChatMessage], temperature: floa def _send_with_retry(self, agent: ModelAgent, payload: dict[str, Any]) -> str: """Call the provider, retrying transient failures with exponential backoff + jitter.""" - last_error: Exception | None = None - for attempt in range(self.max_retries + 1): + attempt = 0 + while True: try: return self._send(agent, payload) except Exception as exc: # noqa: BLE001 - classify then decide - last_error = exc if attempt >= self.max_retries or not is_transient_error(exc): - break + raise RuntimeError(f"provider {agent.id} request failed") from exc self._sleep(self._backoff_delay(attempt)) - raise RuntimeError(f"provider {agent.id} request failed") from last_error + attempt += 1 def _backoff_delay(self, attempt: int) -> float: """Full-jitter exponential backoff, capped, so retries do not thundering-herd a provider.""" @@ -306,12 +346,104 @@ def _send(self, agent: ModelAgent, payload: dict[str, Any]) -> str: return data["choices"][0]["message"]["content"] def _open_provider(self, request: urllib.request.Request) -> Any: - """Open a provider request built from a validated provider URL.""" - return urllib.request.urlopen( # nosec B310 - request URL comes from _provider_url after provider validation. - request, - timeout=self.timeout, - context=self._ssl_context, - ) + """Open one request using only the validation-time provider addresses. + + Public provider methods require HTTPS and call ``_validate_provider`` + first. Plain HTTP remains a narrow private-loopback integration seam; + production provider validation rejects it before credentials are used. + """ + parsed = urlparse(request.full_url) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise RuntimeError("provider request URL must use http(s)") + if parsed.username is not None or parsed.password is not None: + raise RuntimeError("provider request URL must not contain user information") + + target = parsed.path or "/" + if parsed.params: + target = f"{target};{parsed.params}" + if parsed.query: + target = f"{target}?{parsed.query}" + headers = dict(request.header_items()) + headers["Connection"] = "close" + + if parsed.scheme == "http": + if not _literal_loopback_host(parsed.hostname): + raise RuntimeError( + "plain HTTP provider requests require a literal loopback target" + ) + connection = self._http_connection_class( + parsed.hostname, + parsed.port or 80, + timeout=self.timeout, + ) + try: + connection.request( + request.get_method(), + target, + body=request.data, + headers=headers, + ) + response = connection.getresponse() + except (OSError, http.client.HTTPException) as exc: + connection.close() + raise urllib.error.URLError(exc) from exc + if response.status >= 300: + status = response.status + reason = response.reason + response_headers = response.headers + response.close() + connection.close() + raise urllib.error.HTTPError( + request.full_url, + status, + reason, + response_headers, + None, + ) + return _ProviderHTTPResponse(response, connection) + + port = parsed.port or 443 + pin_key = (parsed.hostname.lower(), port) + addresses = getattr(self._local, "provider_address_pins", {}).get(pin_key) + if not addresses: + raise RuntimeError("provider request has no validated address pin") + + last_error: BaseException | None = None + for pinned_ip in addresses: + connection = self._https_connection_class( + parsed.hostname, + pinned_ip, + port, + self.timeout, + self._ssl_context, + ) + try: + connection.request( + request.get_method(), + target, + body=request.data, + headers=headers, + ) + response = connection.getresponse() + except (OSError, http.client.HTTPException) as exc: + connection.close() + last_error = exc + continue + if response.status >= 300: + status = response.status + reason = response.reason + response_headers = response.headers + response.close() + connection.close() + raise urllib.error.HTTPError( + request.full_url, + status, + reason, + response_headers, + None, + ) + return _ProviderHTTPResponse(response, connection) + raise urllib.error.URLError(last_error or "provider connection failed") def stream_chat(self, agent: ModelAgent, messages: list[ChatMessage], temperature: float = 0.2): """Yield content deltas from a mock or OpenAI-compatible streaming endpoint. @@ -358,9 +490,9 @@ def _stream_send(self, agent: ModelAgent, payload: dict[str, Any]): if data == "[DONE]": break try: - chunk = json.loads(data) - except json.JSONDecodeError: - continue + chunk = _parse_provider_json_object_text(data) + except RuntimeError: + raise RuntimeError("malformed provider stream event") from None delta = chunk.get("choices", [{}])[0].get("delta", {}).get("content") if delta: yield delta @@ -451,7 +583,10 @@ def _mock_raw( } def _validate_provider(self, agent: ModelAgent) -> None: - """Reject unsafe remote model endpoints before any egress happens.""" + """Validate one provider and retain its exact approved DNS answer.""" + # Clear every prior thread-local pin before any credential or URL check so + # a failed revalidation can never reuse an earlier approved destination. + self._local.provider_address_pins = {} # Runtime secret must be resolvable from the KV — never an env var name, # never a silent os.getenv fallback. (Legacy api_key_env, if set, is used # only as the credential NAME; see ModelAgent.credential_name.) @@ -471,16 +606,9 @@ def _validate_provider(self, agent: ModelAgent) -> None: hostname = parsed.hostname.lower() if allowed_hosts and hostname not in allowed_hosts: raise RuntimeError(f"{agent.id} provider host is not allowlisted") - for address in socket.getaddrinfo(hostname, parsed.port or 443, type=socket.SOCK_STREAM): - ip_address = ipaddress.ip_address(address[4][0]) - if ( - ip_address.is_private - or ip_address.is_loopback - or ip_address.is_link_local - or ip_address.is_multicast - or ip_address.is_reserved - ): - raise RuntimeError(f"{agent.id} provider resolves to non-public address") + port = parsed.port or 443 + addresses = _validated_public_addresses(hostname, port, agent.id) + self._local.provider_address_pins[(hostname, port)] = addresses def _provider_url(self, agent: ModelAgent, path: str) -> str: """Build a provider URL while rejecting urllib-supported local schemes.""" @@ -855,7 +983,6 @@ def __init__( # Optional durable persistence: default None keeps all state purely in-memory # (zero behavior change). When set, runs/audit/analytics survive restart. self._store = _StateStore(state_db) if state_db else None - self._commercial_report_cache_local = threading.local() if self._store is not None: self._reload_state() @@ -1235,7 +1362,7 @@ def patch_agent(self, agent_pool_id: str, worker_agent_id: str, patch: dict[str, """Apply governance updates to an agent and emit an audit event.""" if not patch: # pragma: no cover raise ValueError("patch request body must contain updates") - if agent_pool_id != "default": # pragma: no cover + if agent_pool_id not in {"default", "default_pool"}: # pragma: no cover raise KeyError(agent_pool_id) current = self._agent(worker_agent_id) patched = current @@ -1287,7 +1414,7 @@ def patch_agent(self, agent_pool_id: str, worker_agent_id: str, patch: dict[str, def add_agent(self, agent_pool_id: str, value: dict[str, Any]) -> dict[str, Any]: """Register a new worker agent (model group member) at runtime; persists when agents_db is set.""" - if agent_pool_id != "default": # pragma: no cover + if agent_pool_id not in {"default", "default_pool"}: # pragma: no cover raise KeyError(agent_pool_id) if "id" not in value or "model" not in value: raise ValueError("agent requires id and model") @@ -1315,7 +1442,7 @@ def add_agent(self, agent_pool_id: str, value: dict[str, Any]) -> dict[str, Any] def remove_agent(self, agent_pool_id: str, worker_agent_id: str) -> dict[str, Any]: """Remove a worker agent from the pool; the pool must keep at least one enabled agent.""" - if agent_pool_id != "default": # pragma: no cover + if agent_pool_id not in {"default", "default_pool"}: # pragma: no cover raise KeyError(agent_pool_id) target = self._agent(worker_agent_id) remaining_enabled = [agent for agent in self.agents if agent.id != worker_agent_id and not agent.disabled] @@ -2316,12 +2443,13 @@ def has_file(path: str) -> bool: ), ] summary = self._buyer_manifest_summary(items) - if summary["by_completion_state"].get("blocked", 0): - manifest_status = "buyer_review_blocked" - elif summary["by_completion_state"].get("warning", 0): - manifest_status = "buyer_review_ready_with_warnings" - else: - manifest_status = "buyer_review_ready" + manifest_status = _classify_commercial_status( + summary["by_completion_state"].get("blocked", 0), + summary["by_completion_state"].get("warning", 0), + blocked_status="buyer_review_blocked", + warning_status="buyer_review_ready_with_warnings", + ready_status="buyer_review_ready", + ) return { "manifest_status": manifest_status, @@ -2478,12 +2606,13 @@ def has_file(path: str) -> bool: ] all_items = included_artifacts + follow_up_items summary = self._buyer_manifest_summary(all_items) - if summary["by_completion_state"].get("blocked", 0): - bundle_status = "buyer_handoff_blocked" - elif summary["by_completion_state"].get("warning", 0): - bundle_status = "buyer_handoff_ready_with_warnings" - else: - bundle_status = "buyer_handoff_ready" + bundle_status = _classify_commercial_status( + summary["by_completion_state"].get("blocked", 0), + summary["by_completion_state"].get("warning", 0), + blocked_status="buyer_handoff_blocked", + warning_status="buyer_handoff_ready_with_warnings", + ready_status="buyer_handoff_ready", + ) return { "bundle_status": bundle_status, @@ -2557,15 +2686,18 @@ def saleability_decision_report( for item in handoff["follow_up_items"] if item["completion_state"] == "warning" ] - if concrete_blockers: - saleability_status = "saleability_blocked" - decision_label = "Blocked by concrete defect" - elif warning_conditions: - saleability_status = "saleability_ready_with_warnings" - decision_label = "Ready for buyer diligence with explicit warnings" - else: - saleability_status = "saleability_ready" - decision_label = "Ready for buyer diligence" + saleability_status = _classify_commercial_status( + len(concrete_blockers), + len(warning_conditions), + blocked_status="saleability_blocked", + warning_status="saleability_ready_with_warnings", + ready_status="saleability_ready", + ) + decision_label = { + "saleability_blocked": "Blocked by concrete defect", + "saleability_ready_with_warnings": "Ready for buyer diligence with explicit warnings", + "saleability_ready": "Ready for buyer diligence", + }[saleability_status] return { "saleability_status": saleability_status, @@ -2778,12 +2910,13 @@ def has_file(path: str) -> bool: export_section_summary = self._buyer_manifest_summary(export_sections) blocked_count = export_section_summary["by_completion_state"]["blocked"] + len(concrete_blockers) warning_count = len(required_external_evidence) - if blocked_count: - export_status = "commercial_export_blocked" - elif warning_count: - export_status = "commercial_export_ready_with_warnings" - else: - export_status = "commercial_export_ready" + export_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_export_blocked", + warning_status="commercial_export_ready_with_warnings", + ready_status="commercial_export_ready", + ) return { "export_status": export_status, @@ -2976,12 +3109,13 @@ def has_file(path: str) -> bool: summary = self._buyer_manifest_summary(all_items) blocked_count = summary["by_completion_state"]["blocked"] + len(concrete_blockers) warning_count = summary["by_completion_state"]["warning"] - if blocked_count: - acceptance_status = "commercial_acceptance_blocked" - elif warning_count: - acceptance_status = "commercial_acceptance_ready_with_warnings" - else: - acceptance_status = "commercial_acceptance_ready" + acceptance_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_acceptance_blocked", + warning_status="commercial_acceptance_ready_with_warnings", + ready_status="commercial_acceptance_ready", + ) return { "acceptance_status": acceptance_status, @@ -3238,12 +3372,13 @@ def has_file(path: str) -> bool: summary = self._buyer_manifest_summary(release_artifacts + external_release_gaps) blocked_count = summary["by_completion_state"]["blocked"] + len(concrete_blockers) warning_count = summary["by_completion_state"]["warning"] - if blocked_count: - release_status = "commercial_release_blocked" - elif warning_count: - release_status = "commercial_release_ready_with_warnings" - else: - release_status = "commercial_release_ready" + release_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_release_blocked", + warning_status="commercial_release_ready_with_warnings", + ready_status="commercial_release_ready", + ) return { "release_status": release_status, @@ -3335,12 +3470,13 @@ def commercial_gap_register_report( }) blocked_count = len(concrete_blockers) + (1 if release_blocked else 0) - if blocked_count: - gap_register_status = "commercial_gap_register_blocked" - elif gap_items: - gap_register_status = "commercial_gap_register_open" - else: - gap_register_status = "commercial_gap_register_clear" + gap_register_status = _classify_commercial_status( + blocked_count, + len(gap_items), + blocked_status="commercial_gap_register_blocked", + warning_status="commercial_gap_register_open", + ready_status="commercial_gap_register_clear", + ) production_gap_count = sum(1 for item in gap_items if item["gap_type"] == "production_evidence_gap") buyer_specific_gap_count = sum(1 for item in gap_items if item["gap_type"] == "buyer_specific_gap") @@ -3533,12 +3669,13 @@ def has_file(path: str) -> bool: buyer_specific_gap_count = 1 if buyer_gap else 0 blocked_count = state_counts.get("blocked", 0) + len(concrete_blockers) warning_count = state_counts.get("warning", 0) - if blocked_count: - procurement_status = "commercial_procurement_blocked" - elif warning_count: - procurement_status = "commercial_procurement_ready_with_warnings" - else: - procurement_status = "commercial_procurement_ready" + procurement_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_procurement_blocked", + warning_status="commercial_procurement_ready_with_warnings", + ready_status="commercial_procurement_ready", + ) return { "procurement_status": procurement_status, @@ -3735,12 +3872,13 @@ def has_file(path: str) -> bool: state_counts = Counter(item["completion_state"] for item in contract_items) blocked_count = state_counts.get("blocked", 0) + len(concrete_blockers) warning_count = state_counts.get("warning", 0) - if blocked_count: - contract_status = "commercial_contract_blocked" - elif warning_count: - contract_status = "commercial_contract_ready_with_warnings" - else: - contract_status = "commercial_contract_ready" + contract_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_contract_blocked", + warning_status="commercial_contract_ready_with_warnings", + ready_status="commercial_contract_ready", + ) return { "contract_status": contract_status, @@ -3934,12 +4072,13 @@ def has_file(path: str) -> bool: state_counts = Counter(item["completion_state"] for item in onboarding_items) blocked_count = state_counts.get("blocked", 0) + len(concrete_blockers) warning_count = state_counts.get("warning", 0) - if blocked_count: - onboarding_status = "commercial_onboarding_blocked" - elif warning_count: - onboarding_status = "commercial_onboarding_ready_with_warnings" - else: - onboarding_status = "commercial_onboarding_ready" + onboarding_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_onboarding_blocked", + warning_status="commercial_onboarding_ready_with_warnings", + ready_status="commercial_onboarding_ready", + ) return { "onboarding_status": onboarding_status, @@ -4141,12 +4280,13 @@ def has_file(path: str) -> bool: production_evidence_action_count = sum( 1 for item in operations_items if item.get("source_gap_status") == "production_input_required" ) - if blocked_count: - operations_status = "commercial_operations_blocked" - elif warning_count: - operations_status = "commercial_operations_ready_with_warnings" - else: - operations_status = "commercial_operations_ready" + operations_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_operations_blocked", + warning_status="commercial_operations_ready_with_warnings", + ready_status="commercial_operations_ready", + ) return { "operations_status": operations_status, @@ -4372,12 +4512,13 @@ def has_file(path: str) -> bool: buyer_privacy_gap_count = sum( 1 for item in security_attestation_items if item.get("source_gap_status") == "buyer_input_required" ) - if blocked_count: - security_attestation_status = "commercial_security_attestation_blocked" - elif warning_count: - security_attestation_status = "commercial_security_attestation_ready_with_warnings" - else: - security_attestation_status = "commercial_security_attestation_ready" + security_attestation_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_security_attestation_blocked", + warning_status="commercial_security_attestation_ready_with_warnings", + ready_status="commercial_security_attestation_ready", + ) return { "security_attestation_status": security_attestation_status, @@ -4616,12 +4757,13 @@ def has_file(path: str) -> bool: external_value_proof_gap_count = sum( 1 for item in value_items if item.get("source_gap_status") == "external_value_proof_required" ) - if blocked_count: - value_status = "commercial_value_blocked" - elif warning_count: - value_status = "commercial_value_ready_with_warnings" - else: - value_status = "commercial_value_ready" + value_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_value_blocked", + warning_status="commercial_value_ready_with_warnings", + ready_status="commercial_value_ready", + ) return { "value_status": value_status, @@ -4725,7 +4867,7 @@ def has_file(path: str) -> bool: *operations["concrete_blockers"], *export["concrete_blockers"], ] - concrete_blockers = list(dict.fromkeys(concrete_blockers)) + concrete_blockers = _deduplicate_report_values(concrete_blockers) close_items = [ { "item_name": "sellable_product_packet", @@ -4896,12 +5038,13 @@ def has_file(path: str) -> bool: buyer_signature_gap_count = sum( 1 for item in close_items if item.get("source_gap_status") == "buyer_signature_required" ) - if blocked_count: - close_status = "commercial_close_blocked" - elif warning_count: - close_status = "commercial_close_ready_with_warnings" - else: - close_status = "commercial_close_ready" + close_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_close_blocked", + warning_status="commercial_close_ready_with_warnings", + ready_status="commercial_close_ready", + ) return { "close_status": close_status, @@ -5008,7 +5151,7 @@ def has_file(path: str) -> bool: *export["concrete_blockers"], *saleability["concrete_blockers"], ] - concrete_blockers = list(dict.fromkeys(concrete_blockers)) + concrete_blockers = _deduplicate_report_values(concrete_blockers) gtm_items = [ { "item_name": "commercial_close_packet", @@ -5205,12 +5348,13 @@ def has_file(path: str) -> bool: + value["value_summary"]["external_value_proof_gap_count"] + export["export_summary"]["warning_count"] ) - if blocked_count: - gtm_status = "commercial_go_to_market_blocked" - elif warning_count: - gtm_status = "commercial_go_to_market_ready_with_warnings" - else: - gtm_status = "commercial_go_to_market_ready" + gtm_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_go_to_market_blocked", + warning_status="commercial_go_to_market_ready_with_warnings", + ready_status="commercial_go_to_market_ready", + ) return { "go_to_market_status": gtm_status, @@ -5308,7 +5452,7 @@ def has_file(path: str) -> bool: *onboarding["concrete_blockers"], *acceptance["concrete_blockers"], ] - concrete_blockers = list(dict.fromkeys(concrete_blockers)) + concrete_blockers = _deduplicate_report_values(concrete_blockers) launch_items = [ { "item_name": "go_to_market_packet", @@ -5511,12 +5655,13 @@ def has_file(path: str) -> bool: external_input_group_count = ( buyer_environment_gap_count + production_telemetry_gap_count + commercial_signature_gap_count ) - if blocked_count: - launch_status = "commercial_launch_blocked" - elif warning_count: - launch_status = "commercial_launch_ready_with_warnings" - else: - launch_status = "commercial_launch_ready" + launch_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_launch_blocked", + warning_status="commercial_launch_ready_with_warnings", + ready_status="commercial_launch_ready", + ) return { "launch_status": launch_status, @@ -5609,7 +5754,7 @@ def has_file(path: str) -> bool: concrete_blockers.append("commercial_readiness_failed") if launch["launch_status"] == "commercial_launch_blocked": concrete_blockers.append("commercial_launch_blocked") - concrete_blockers = list(dict.fromkeys(concrete_blockers)) + concrete_blockers = _deduplicate_report_values(concrete_blockers) scorecard_items = [ { "item_name": "product_design_evidence", @@ -5787,12 +5932,13 @@ def has_file(path: str) -> bool: state_counts = Counter(item["completion_state"] for item in scorecard_items) blocked_count = state_counts.get("blocked", 0) + len(concrete_blockers) warning_count = state_counts.get("warning", 0) - if blocked_count: - completion_status = "commercial_completion_blocked" - elif warning_count: - completion_status = "commercial_completion_ready_with_warnings" - else: - completion_status = "commercial_completion_ready" + completion_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_completion_blocked", + warning_status="commercial_completion_ready_with_warnings", + ready_status="commercial_completion_ready", + ) return { "completion_status": completion_status, @@ -5904,7 +6050,7 @@ def step( "next_action": next_action, } - concrete_blockers = list(dict.fromkeys(acceptance["concrete_blockers"] + completion["concrete_blockers"])) + concrete_blockers = _deduplicate_report_values(acceptance["concrete_blockers"] + completion["concrete_blockers"]) acceptance_blocked = acceptance["acceptance_status"] == "commercial_acceptance_blocked" completion_blocked = completion["completion_status"] == "commercial_completion_blocked" local_runtime_state = "blocked" if acceptance_blocked or completion_blocked or concrete_blockers else "ready" @@ -6031,12 +6177,13 @@ def step( state_counts = Counter(item["completion_state"] for item in workflow_steps) blocked_count = state_counts.get("blocked", 0) + len(concrete_blockers) warning_count = state_counts.get("warning", 0) - if blocked_count: - workflow_status = "buyer_acceptance_workflow_blocked" - elif warning_count: - workflow_status = "buyer_acceptance_workflow_ready_with_warnings" - else: - workflow_status = "buyer_acceptance_workflow_ready" + workflow_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="buyer_acceptance_workflow_blocked", + warning_status="buyer_acceptance_workflow_ready_with_warnings", + ready_status="buyer_acceptance_workflow_ready", + ) return { "workflow_status": workflow_status, @@ -6152,9 +6299,7 @@ def step( "expected_evidence": expected_evidence, } - concrete_blockers = list( - dict.fromkeys(completion["concrete_blockers"] + buyer_workflow["concrete_blockers"]) - ) + concrete_blockers = _deduplicate_report_values(completion["concrete_blockers"] + buyer_workflow["concrete_blockers"]) local_runtime_state = ( "blocked" if completion["completion_status"] == "commercial_completion_blocked" @@ -6302,12 +6447,13 @@ def step( state_counts = Counter(item["completion_state"] for item in demo_steps) blocked_count = state_counts.get("blocked", 0) + len(concrete_blockers) warning_count = state_counts.get("warning", 0) - if blocked_count: - demo_status = "commercial_demo_blocked" - elif warning_count: - demo_status = "commercial_demo_ready_with_warnings" - else: - demo_status = "commercial_demo_ready" + demo_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_demo_blocked", + warning_status="commercial_demo_ready_with_warnings", + ready_status="commercial_demo_ready", + ) required_runtime_endpoints = list( dict.fromkeys( endpoint @@ -6470,13 +6616,9 @@ def section( "next_action": next_action, } - concrete_blockers = list( - dict.fromkeys( - completion["concrete_blockers"] + concrete_blockers = _deduplicate_report_values(completion["concrete_blockers"] + demo["concrete_blockers"] - + buyer_workflow["concrete_blockers"] - ) - ) + + buyer_workflow["concrete_blockers"]) local_runtime_state = ( "blocked" if completion["completion_status"] == "commercial_completion_blocked" @@ -6649,12 +6791,13 @@ def section( state_counts = Counter(item["completion_state"] for item in proposal_sections) blocked_count = state_counts.get("blocked", 0) + len(concrete_blockers) warning_count = state_counts.get("warning", 0) - if blocked_count: - proposal_status = "commercial_proposal_blocked" - elif warning_count: - proposal_status = "commercial_proposal_ready_with_warnings" - else: - proposal_status = "commercial_proposal_ready" + proposal_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_proposal_blocked", + warning_status="commercial_proposal_ready_with_warnings", + ready_status="commercial_proposal_ready", + ) required_runtime_endpoints = list( dict.fromkeys( endpoint @@ -6823,7 +6966,7 @@ def gate( "next_action": next_action, } - concrete_blockers = list(dict.fromkeys(proposal["concrete_blockers"] + close["concrete_blockers"])) + concrete_blockers = _deduplicate_report_values(proposal["concrete_blockers"] + close["concrete_blockers"]) local_runtime_state = ( "blocked" if proposal["proposal_status"] == "commercial_proposal_blocked" @@ -6992,12 +7135,13 @@ def gate( state_counts = Counter(item["completion_state"] for item in approval_gates) blocked_count = state_counts.get("blocked", 0) + len(concrete_blockers) warning_count = state_counts.get("warning", 0) - if blocked_count: - purchase_approval_status = "commercial_purchase_approval_blocked" - elif warning_count: - purchase_approval_status = "commercial_purchase_approval_ready_with_warnings" - else: - purchase_approval_status = "commercial_purchase_approval_ready" + purchase_approval_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_purchase_approval_blocked", + warning_status="commercial_purchase_approval_ready_with_warnings", + ready_status="commercial_purchase_approval_ready", + ) required_runtime_endpoints = list( dict.fromkeys( endpoint @@ -7187,15 +7331,11 @@ def section( "next_action": next_action, } - concrete_blockers = list( - dict.fromkeys( - purchase["concrete_blockers"] + concrete_blockers = _deduplicate_report_values(purchase["concrete_blockers"] + proposal["concrete_blockers"] + completion["concrete_blockers"] + demo["concrete_blockers"] - + buyer_workflow["concrete_blockers"] - ) - ) + + buyer_workflow["concrete_blockers"]) local_runtime_state = ( "blocked" if purchase["purchase_approval_status"] == "commercial_purchase_approval_blocked" @@ -7391,12 +7531,13 @@ def section( state_counts = Counter(item["completion_state"] for item in diligence_sections) blocked_count = state_counts.get("blocked", 0) + len(concrete_blockers) warning_count = state_counts.get("warning", 0) - if blocked_count: - due_diligence_status = "commercial_due_diligence_blocked" - elif warning_count: - due_diligence_status = "commercial_due_diligence_ready_with_warnings" - else: - due_diligence_status = "commercial_due_diligence_ready" + due_diligence_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_due_diligence_blocked", + warning_status="commercial_due_diligence_ready_with_warnings", + ready_status="commercial_due_diligence_ready", + ) required_runtime_endpoints = list( dict.fromkeys( endpoint @@ -7604,16 +7745,12 @@ def section( "next_action": next_action, } - concrete_blockers = list( - dict.fromkeys( - due_diligence["concrete_blockers"] + concrete_blockers = _deduplicate_report_values(due_diligence["concrete_blockers"] + purchase["concrete_blockers"] + proposal["concrete_blockers"] + completion["concrete_blockers"] + demo["concrete_blockers"] - + buyer_workflow["concrete_blockers"] - ) - ) + + buyer_workflow["concrete_blockers"]) local_runtime_state = ( "blocked" if due_diligence["due_diligence_status"] == "commercial_due_diligence_blocked" @@ -7792,15 +7929,18 @@ def section( state_counts = Counter(item["completion_state"] for item in memo_sections) blocked_count = state_counts.get("blocked", 0) + len(concrete_blockers) warning_count = state_counts.get("warning", 0) - if blocked_count: - investment_committee_status = "commercial_investment_committee_blocked" - recommendation_status = "do_not_recommend_until_blockers_cleared" - elif warning_count: - investment_committee_status = "commercial_investment_committee_ready_with_warnings" - recommendation_status = "recommend_with_buyer_conditions" - else: - investment_committee_status = "commercial_investment_committee_ready" - recommendation_status = "recommend" + investment_committee_status = _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="commercial_investment_committee_blocked", + warning_status="commercial_investment_committee_ready_with_warnings", + ready_status="commercial_investment_committee_ready", + ) + recommendation_status = { + "commercial_investment_committee_blocked": "do_not_recommend_until_blockers_cleared", + "commercial_investment_committee_ready_with_warnings": "recommend_with_buyer_conditions", + "commercial_investment_committee_ready": "recommend", + }[investment_committee_status] required_runtime_endpoints = list( dict.fromkeys( endpoint @@ -8162,44 +8302,6 @@ def _criteria_summary(self, criteria: list[dict[str, str]]) -> dict[str, int]: } -def _report_cache_token(value: Any) -> Any: - if isinstance(value, (str, int, float, bool, type(None))): - return value - if isinstance(value, tuple): - return tuple(_report_cache_token(item) for item in value) - return ("id", id(value)) - - -def _commercial_report_cached(method: Any) -> Any: - @wraps(method) - def wrapper(self: TaskOrchestrator, *args: Any, **kwargs: Any) -> dict[str, Any]: - local = self._commercial_report_cache_local - depth = getattr(local, "depth", 0) - if depth == 0: - local.cache = {} - local.depth = depth + 1 - try: - key = ( - method.__name__, - _report_cache_token(args), - tuple(sorted((name, _report_cache_token(value)) for name, value in kwargs.items())), - ) - if key not in local.cache: - local.cache[key] = method(self, *args, **kwargs) - return local.cache[key] - finally: - local.depth -= 1 - if depth == 0: - local.cache = {} - - return wrapper - - -for _report_name, _report_method in list(TaskOrchestrator.__dict__.items()): - if _report_name.startswith("commercial_") and _report_name.endswith("_report"): - setattr(TaskOrchestrator, _report_name, _commercial_report_cached(_report_method)) - - def redact_text(text: str) -> str: """Mask common secret and personal-data shapes from traces.""" redacted = text @@ -8242,6 +8344,15 @@ def _freeze_report_cache_value(value: Any) -> Any: return value +def _deduplicate_report_values(values: list[Any]) -> list[Any]: + """Return first-occurrence report values without requiring hashability.""" + unique_values: list[Any] = [] + for value in values: + if value not in unique_values: + unique_values.append(value) + return unique_values + + def _cached_commercial_report(method: Any) -> Any: @wraps(method) def wrapper(self: TaskOrchestrator, *args: Any, **kwargs: Any) -> dict[str, Any]: diff --git a/contextual_orchestrator/provider_catalog.py b/contextual_orchestrator/provider_catalog.py new file mode 100644 index 000000000..568738321 --- /dev/null +++ b/contextual_orchestrator/provider_catalog.py @@ -0,0 +1,1208 @@ +"""Durable provider discovery, catalog persistence, and agent-pool construction. + +GitHub Actions or another trusted bootstrap process may transport the fixed +provider credential inventory into :mod:`contextual_orchestrator.credentials`. +Runtime inference resolves credential names from that registry and never reads +provider API keys directly from ambient environment variables. + +Provider metadata is stored separately from secret values in a normalized +catalog. Account refreshes are isolated: a failed refresh preserves that +account's last-known-good models, while a first deployment with no usable model +fails closed instead of silently starting a mock or empty pool. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import hashlib +import http.client +import ipaddress +import json +import math +import os +import random +import re +import socket +import ssl +import sys +import time +from typing import Any, Callable, Iterable, Mapping, Protocol, Sequence +from urllib.parse import quote, urlparse + +from .credentials import get_credential, register_credential +from .orchestrator import ModelAgent, ModelClient, TaskOrchestrator + + +CATALOG_RESPONSE_MAX_BYTES = 8 * 1024 * 1024 +"""Maximum accepted bytes in one provider model-catalog response.""" + +PROVIDER_CATALOG_SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS provider_accounts ( + provider_account_id text PRIMARY KEY, + provider_name text NOT NULL, + credential_name text NOT NULL, + base_url text NOT NULL, + models_path text, + transport_name text NOT NULL, + auth_header_name text NOT NULL, + auth_prefix text NOT NULL, + enabled_flag boolean NOT NULL DEFAULT true, + priority_rank integer NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS provider_models ( + provider_model_id text PRIMARY KEY, + provider_account_id text NOT NULL REFERENCES provider_accounts(provider_account_id), + model_name text NOT NULL, + display_name text NOT NULL, + context_window integer, + input_price_usd_per_million numeric(20, 8), + output_price_usd_per_million numeric(20, 8), + enabled_flag boolean NOT NULL DEFAULT true, + first_discovered_at timestamptz NOT NULL, + last_seen_at timestamptz NOT NULL, + UNIQUE (provider_account_id, model_name) +); + +CREATE TABLE IF NOT EXISTS model_capabilities ( + provider_model_id text NOT NULL REFERENCES provider_models(provider_model_id) ON DELETE CASCADE, + capability_name text NOT NULL, + PRIMARY KEY (provider_model_id, capability_name) +); + +CREATE TABLE IF NOT EXISTS model_modalities ( + provider_model_id text NOT NULL REFERENCES provider_models(provider_model_id) ON DELETE CASCADE, + modality_name text NOT NULL, + PRIMARY KEY (provider_model_id, modality_name) +); + +CREATE TABLE IF NOT EXISTS catalog_refresh_runs ( + catalog_refresh_id text PRIMARY KEY, + provider_account_id text NOT NULL REFERENCES provider_accounts(provider_account_id), + refresh_status text NOT NULL, + observed_model_count integer NOT NULL DEFAULT 0, + error_code text, + started_at timestamptz NOT NULL, + finished_at timestamptz NOT NULL +); + +CREATE INDEX IF NOT EXISTS provider_models_account_idx + ON provider_models (provider_account_id, enabled_flag); +CREATE INDEX IF NOT EXISTS catalog_refresh_account_idx + ON catalog_refresh_runs (provider_account_id, finished_at DESC); +""" +"""Normalized PostgreSQL schema for provider accounts, models, and refresh evidence.""" + + +@dataclass(frozen=True) +class ProviderAccount: + """One independently governed provider account and credential reference.""" + + provider_account_id: str + provider_name: str + credential_name: str + base_url: str + models_path: str | None = "/models" + transport_name: str = "openai_compatible" + auth_header_name: str = "Authorization" + auth_prefix: str = "Bearer" + enabled: bool = True + priority_rank: int = 0 + + @property + def models_url(self) -> str | None: + """Return the complete model-list endpoint, or ``None`` when unsupported.""" + if self.models_path is None: + return None + return f"{self.base_url.rstrip('/')}/{self.models_path.lstrip('/')}" + + +@dataclass(frozen=True) +class DiscoveredModel: + """Provider-neutral metadata for one discovered model identifier.""" + + model_name: str + display_name: str + capabilities: tuple[str, ...] = ("chat",) + modalities: tuple[str, ...] = ("text",) + context_window: int | None = None + input_price_usd_per_million: float | None = None + output_price_usd_per_million: float | None = None + + +@dataclass(frozen=True) +class CatalogModelRecord: + """A discovered model associated with its provider account.""" + + provider_account_id: str + model: DiscoveredModel + + +class ProviderCatalogUnavailable(RuntimeError): + """Raised when a durable catalog cannot produce any usable provider model.""" + + +class CatalogHttpError(RuntimeError): + """Stable, secret-free provider catalog transport failure.""" + + def __init__(self, code: str, *, transient: bool = False) -> None: + super().__init__(code) + self.code = code + self.transient = transient + + +class ProviderCatalogStore(Protocol): + """Persistence contract shared by in-memory tests and PostgreSQL production.""" + + def upsert_account(self, account: ProviderAccount) -> None: + """Insert or update one provider account without storing a secret value.""" + ... + + def replace_catalog(self, account: ProviderAccount, models: Sequence[DiscoveredModel]) -> None: + """Atomically replace one successful provider account's current model set.""" + ... + + def record_failure(self, account: ProviderAccount, error_code: str) -> None: + """Record a failed refresh without changing the last-known-good model set.""" + ... + + def enabled_models(self) -> list[CatalogModelRecord]: + """Return usable models belonging to enabled provider accounts.""" + ... + + def all_models(self) -> list[CatalogModelRecord]: + """Return catalog history including models on disabled accounts.""" + ... + + def has_models(self, provider_account_id: str) -> bool: + """Return whether an account retains any enabled last-known-good model.""" + ... + + +DEFAULT_PROVIDER_ACCOUNTS: tuple[ProviderAccount, ...] = ( + ProviderAccount( + provider_account_id="nvidia_nim_primary", + provider_name="nvidia_nim", + credential_name="NVIDIA_NIM_API_KEY", + base_url="https://integrate.api.nvidia.com/v1", + priority_rank=1, + ), + ProviderAccount( + provider_account_id="nvidia_nim_secondary", + provider_name="nvidia_nim", + credential_name="NVIDIA_NIM_API_KEY_SUB", + base_url="https://integrate.api.nvidia.com/v1", + priority_rank=0, + ), + ProviderAccount( + provider_account_id="bytez_primary", + provider_name="bytez", + credential_name="BYTEZ_API_KEY", + base_url="https://api.bytez.com", + models_path="/models/v2", + transport_name="bytez_v2", + auth_prefix="Key", + priority_rank=0, + ), + ProviderAccount( + provider_account_id="openrouter_primary", + provider_name="openrouter", + credential_name="OPENROUTER_API_KEY", + base_url="https://openrouter.ai/api/v1", + priority_rank=1, + ), + ProviderAccount( + provider_account_id="openai_primary", + provider_name="openai", + credential_name="OPENAI_API_KEY", + base_url="https://api.openai.com/v1", + priority_rank=1, + ), +) +"""Fixed bootstrap inventory corresponding to the five organization secrets.""" + + +class InMemoryProviderCatalogStore: + """Deterministic catalog store for tests and standalone evaluation.""" + + def __init__(self) -> None: + self._accounts: dict[str, ProviderAccount] = {} + self._models: dict[str, dict[str, DiscoveredModel]] = {} + self.refresh_runs: list[dict[str, Any]] = [] + + def upsert_account(self, account: ProviderAccount) -> None: + """Store an account definition, preserving its model history.""" + self._accounts[account.provider_account_id] = account + + def replace_catalog(self, account: ProviderAccount, models: Sequence[DiscoveredModel]) -> None: + """Replace one account catalog and append successful refresh evidence.""" + self.upsert_account(account) + unique = {model.model_name: model for model in models if model.model_name} + self._models[account.provider_account_id] = unique + now = _utc_now_text() + self.refresh_runs.append( + { + "catalog_refresh_id": _refresh_id(account.provider_account_id, now), + "provider_account_id": account.provider_account_id, + "refresh_status": "refreshed", + "observed_model_count": len(unique), + "error_code": None, + "started_at": now, + "finished_at": now, + } + ) + + def record_failure(self, account: ProviderAccount, error_code: str) -> None: + """Append failure evidence while leaving the prior model mapping unchanged.""" + self.upsert_account(account) + now = _utc_now_text() + self.refresh_runs.append( + { + "catalog_refresh_id": _refresh_id(account.provider_account_id, now), + "provider_account_id": account.provider_account_id, + "refresh_status": "failed", + "observed_model_count": 0, + "error_code": error_code, + "started_at": now, + "finished_at": now, + } + ) + + def enabled_models(self) -> list[CatalogModelRecord]: + """Return sorted model rows whose provider account is enabled.""" + records: list[CatalogModelRecord] = [] + for account_id, models in self._models.items(): + if not self._accounts[account_id].enabled: + continue + records.extend(CatalogModelRecord(account_id, model) for model in models.values()) + return sorted(records, key=lambda row: (row.provider_account_id, row.model.model_name)) + + def all_models(self) -> list[CatalogModelRecord]: + """Return every retained model regardless of account enablement.""" + return sorted( + ( + CatalogModelRecord(account_id, model) + for account_id, models in self._models.items() + for model in models.values() + ), + key=lambda row: (row.provider_account_id, row.model.model_name), + ) + + def has_models(self, provider_account_id: str) -> bool: + """Return whether an account has at least one retained model.""" + return bool(self._models.get(provider_account_id)) + + +class PostgresProviderCatalogStore: # pragma: no cover - production database adapter + """Normalized PostgreSQL provider catalog with account-scoped transactions.""" + + def __init__(self, dsn: str) -> None: + if not dsn: + raise ProviderCatalogUnavailable("provider catalog requires a PostgreSQL DSN") + self._dsn = dsn + self._schema_ready = False + + def _connect(self): + try: + import psycopg + except ImportError as exc: + raise ProviderCatalogUnavailable( + "provider catalog requires contextual-orchestrator[db]" + ) from exc + return psycopg.connect(self._dsn) + + def _ensure_schema(self, connection: Any) -> None: + if self._schema_ready: + return + with connection.cursor() as cursor: + cursor.execute(PROVIDER_CATALOG_SCHEMA_SQL) + connection.commit() + self._schema_ready = True + + def upsert_account(self, account: ProviderAccount) -> None: + """Create or update one provider account in the catalog.""" + with self._connect() as connection: + self._ensure_schema(connection) + with connection.cursor() as cursor: + _upsert_account_row(cursor, account) + connection.commit() + + def replace_catalog(self, account: ProviderAccount, models: Sequence[DiscoveredModel]) -> None: + """Replace the account's enabled models and record a refresh.""" + started_at = _utc_now() + unique = {model.model_name: model for model in models if model.model_name} + with self._connect() as connection: + self._ensure_schema(connection) + with connection.cursor() as cursor: + _upsert_account_row(cursor, account) + seen_ids: list[str] = [] + for model in unique.values(): + model_id = _provider_model_id(account.provider_account_id, model.model_name) + seen_ids.append(model_id) + cursor.execute( + "INSERT INTO provider_models (" + "provider_model_id, provider_account_id, model_name, display_name, " + "context_window, input_price_usd_per_million, output_price_usd_per_million, " + "enabled_flag, first_discovered_at, last_seen_at) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, true, %s, %s) " + "ON CONFLICT (provider_model_id) DO UPDATE SET " + "display_name = EXCLUDED.display_name, context_window = EXCLUDED.context_window, " + "input_price_usd_per_million = EXCLUDED.input_price_usd_per_million, " + "output_price_usd_per_million = EXCLUDED.output_price_usd_per_million, " + "enabled_flag = true, last_seen_at = EXCLUDED.last_seen_at", + ( + model_id, + account.provider_account_id, + model.model_name, + model.display_name, + model.context_window, + model.input_price_usd_per_million, + model.output_price_usd_per_million, + started_at, + started_at, + ), + ) + cursor.execute( + "DELETE FROM model_capabilities WHERE provider_model_id = %s", + (model_id,), + ) + cursor.execute( + "DELETE FROM model_modalities WHERE provider_model_id = %s", + (model_id,), + ) + for capability in model.capabilities: + cursor.execute( + "INSERT INTO model_capabilities (provider_model_id, capability_name) " + "VALUES (%s, %s) ON CONFLICT DO NOTHING", + (model_id, capability), + ) + for modality in model.modalities: + cursor.execute( + "INSERT INTO model_modalities (provider_model_id, modality_name) " + "VALUES (%s, %s) ON CONFLICT DO NOTHING", + (model_id, modality), + ) + if seen_ids: + cursor.execute( + "UPDATE provider_models SET enabled_flag = false " + "WHERE provider_account_id = %s AND NOT (provider_model_id = ANY(%s))", + (account.provider_account_id, seen_ids), + ) + else: + cursor.execute( + "UPDATE provider_models SET enabled_flag = false " + "WHERE provider_account_id = %s", + (account.provider_account_id,), + ) + _insert_refresh_row( + cursor, + account.provider_account_id, + "refreshed", + len(unique), + None, + started_at, + _utc_now(), + ) + connection.commit() + + def record_failure(self, account: ProviderAccount, error_code: str) -> None: + """Record a failed catalog refresh for one provider account.""" + started_at = _utc_now() + with self._connect() as connection: + self._ensure_schema(connection) + with connection.cursor() as cursor: + _upsert_account_row(cursor, account) + _insert_refresh_row( + cursor, + account.provider_account_id, + "failed", + 0, + error_code, + started_at, + _utc_now(), + ) + connection.commit() + + def enabled_models(self) -> list[CatalogModelRecord]: + """Return models from enabled provider accounts.""" + return self._read_models(enabled_accounts_only=True) + + def all_models(self) -> list[CatalogModelRecord]: + """Return models from enabled and disabled provider accounts.""" + return self._read_models(enabled_accounts_only=False) + + def has_models(self, provider_account_id: str) -> bool: + """Report whether an account has at least one enabled model.""" + with self._connect() as connection: + self._ensure_schema(connection) + with connection.cursor() as cursor: + cursor.execute( + "SELECT EXISTS (SELECT 1 FROM provider_models " + "WHERE provider_account_id = %s AND enabled_flag = true)", + (provider_account_id,), + ) + row = cursor.fetchone() + return bool(row and row[0]) + + def _read_models(self, *, enabled_accounts_only: bool) -> list[CatalogModelRecord]: + condition = "AND a.enabled_flag = true" if enabled_accounts_only else "" + with self._connect() as connection: + self._ensure_schema(connection) + with connection.cursor() as cursor: + cursor.execute( + "SELECT m.provider_account_id, m.provider_model_id, m.model_name, " + "m.display_name, m.context_window, m.input_price_usd_per_million, " + "m.output_price_usd_per_million " + "FROM provider_models m JOIN provider_accounts a " + "ON a.provider_account_id = m.provider_account_id " + f"WHERE m.enabled_flag = true {condition} " # nosec B608 - fixed fragment + "ORDER BY m.provider_account_id, m.model_name" + ) + rows = cursor.fetchall() + records: list[CatalogModelRecord] = [] + for row in rows: + cursor.execute( + "SELECT capability_name FROM model_capabilities " + "WHERE provider_model_id = %s ORDER BY capability_name", + (row[1],), + ) + capabilities = tuple(item[0] for item in cursor.fetchall()) + cursor.execute( + "SELECT modality_name FROM model_modalities " + "WHERE provider_model_id = %s ORDER BY modality_name", + (row[1],), + ) + modalities = tuple(item[0] for item in cursor.fetchall()) + records.append( + CatalogModelRecord( + row[0], + DiscoveredModel( + model_name=row[2], + display_name=row[3], + capabilities=capabilities, + modalities=modalities, + context_window=row[4], + input_price_usd_per_million=_optional_float(row[5]), + output_price_usd_per_million=_optional_float(row[6]), + ), + ) + ) + return records + + +class _PinnedCatalogConnection(http.client.HTTPSConnection): # pragma: no cover - network adapter + """Connect to a validated address while retaining hostname TLS verification.""" + + def __init__(self, hostname: str, pinned_ip: str, port: int, timeout: float, context: ssl.SSLContext) -> None: + super().__init__(hostname, port=port, timeout=timeout, context=context) + self._pinned_ip = pinned_ip + self._catalog_hostname = hostname + + def connect(self) -> None: + raw_socket = socket.create_connection((self._pinned_ip, self.port), self.timeout) + try: + self.sock = self._context.wrap_socket(raw_socket, server_hostname=self._catalog_hostname) + except Exception: + raw_socket.close() + raise + + +class ProviderCatalogHttpClient: + """Bounded DNS-pinned HTTPS client for provider model listings.""" + + TRANSIENT_STATUS = frozenset({408, 409, 425, 429, 500, 502, 503, 504}) + + def __init__( + self, + *, + timeout_seconds: float = 20.0, + max_attempts: int = 3, + deadline_seconds: float = 60.0, + sleep: Callable[[float], None] = time.sleep, + random_uniform: Callable[[float, float], float] = random.uniform, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if timeout_seconds <= 0 or max_attempts < 1 or deadline_seconds <= 0: + raise ValueError("catalog HTTP limits must be positive") + self.timeout_seconds = timeout_seconds + self.max_attempts = max_attempts + self.deadline_seconds = deadline_seconds + self._sleep = sleep + self._random_uniform = random_uniform + self._clock = clock + self._ssl_context = ssl.create_default_context() + + def discover(self, account: ProviderAccount, credential: str) -> list[DiscoveredModel]: + """Fetch and normalize one account's model document with bounded retries.""" + if account.models_url is None: + raise CatalogHttpError("catalog_endpoint_not_configured") + started = self._clock() + for attempt in range(self.max_attempts): + if self._clock() - started >= self.deadline_seconds: + raise CatalogHttpError("catalog_deadline_exceeded", transient=True) + try: + document = self._request_json(account, credential) + models = normalize_models_document(document) + if not models: + raise CatalogHttpError("catalog_contains_no_models") + return models + except CatalogHttpError as exc: + if not exc.transient or attempt + 1 >= self.max_attempts: + raise + ceiling = min(8.0, 0.5 * (2**attempt)) + self._sleep(self._random_uniform(0.0, ceiling)) + raise CatalogHttpError("catalog_attempts_exhausted", transient=True) + + def _request_json(self, account: ProviderAccount, credential: str) -> dict[str, Any]: # pragma: no cover - network + return _secure_json_request( + method="GET", + url=account.models_url or "", + header_name=account.auth_header_name, + authorization=f"{account.auth_prefix} {credential}".strip(), + payload=None, + timeout_seconds=self.timeout_seconds, + transient_status=self.TRANSIENT_STATUS, + ) + + +class ProviderAwareModelClient(ModelClient): + """Use the existing OpenAI transport plus a narrow native Bytez adapter.""" + + def __init__( + self, + *args: Any, + bytez_request: Callable[[ModelAgent, list[dict[str, str]], str], Mapping[str, Any]] | None = None, + **kwargs: Any, + ) -> None: + super().__init__(*args, **kwargs) + self._bytez_request = bytez_request or self._request_bytez + + def chat(self, agent: ModelAgent, messages: list[dict[str, str]], temperature: float = 0.2) -> str: + """Dispatch Bytez through its native Key/input contract and delegate all peers.""" + if agent.provider_name != "bytez": + return super().chat(agent, messages, temperature=temperature) + self._local.usage = None + credential = get_credential(agent.credential_name) + if not credential: + raise ProviderCatalogUnavailable("Bytez credential is not registered") + document = self._bytez_request(agent, messages, credential) + return _normalize_bytez_output(document) + + def stream_chat(self, agent: ModelAgent, messages: list[dict[str, str]], temperature: float = 0.2): + """Frame a completed native Bytez answer when that API offers no token SSE contract.""" + if agent.provider_name != "bytez": + yield from super().stream_chat(agent, messages, temperature=temperature) + return + answer = self.chat(agent, messages, temperature=temperature) + for start in range(0, len(answer), 24): + yield answer[start : start + 24] + + def proxy_send(self, agent: ModelAgent, endpoint: str, payload: dict[str, Any]) -> dict[str, Any]: + """Fail closed for unsupported Bytez passthrough instead of fabricating OpenAI shapes.""" + if agent.provider_name == "bytez": + raise ProviderCatalogUnavailable( + f"Bytez native transport does not support passthrough endpoint {endpoint}" + ) + return super().proxy_send(agent, endpoint, payload) + + def _request_bytez( + self, + agent: ModelAgent, + messages: list[dict[str, str]], + credential: str, + ) -> Mapping[str, Any]: # pragma: no cover - real Bytez network boundary + model_path = quote(agent.model, safe="") + return _secure_json_request( + method="POST", + url=f"{agent.base_url.rstrip('/')}/models/v2/{model_path}", + header_name="Authorization", + authorization=f"Key {credential}", + payload={"input": messages}, + timeout_seconds=float(self.timeout), + transient_status=ProviderCatalogHttpClient.TRANSIENT_STATUS, + ) + + +class ProviderCatalogService: + """Coordinate isolated provider refreshes and build the runtime agent pool.""" + + def __init__( + self, + *, + store: ProviderCatalogStore, + accounts: Sequence[ProviderAccount] = DEFAULT_PROVIDER_ACCOUNTS, + discover: Callable[[ProviderAccount, str], Sequence[DiscoveredModel]] | None = None, + ) -> None: + self.store = store + self.accounts = tuple(accounts) + self._account_by_id = {account.provider_account_id: account for account in self.accounts} + self._discover = discover or ProviderCatalogHttpClient().discover + self.last_refresh_summary: dict[str, Any] = { + "provider_accounts": {}, + "candidate_model_count": 0, + "measurement_status": "provider_catalog_snapshot", + } + + def refresh_all(self) -> dict[str, Any]: + """Refresh each account independently and preserve stale usable catalogs.""" + provider_rows: dict[str, dict[str, Any]] = {} + for account in self.accounts: + self.store.upsert_account(account) + if not account.enabled: + provider_rows[account.provider_account_id] = { + "status": "disabled", + "model_count": 0, + "error_code": None, + } + continue + credential = get_credential(account.credential_name) + if not credential: + provider_rows[account.provider_account_id] = self._failed_refresh( + account, "credential_not_registered" + ) + continue + try: + models = list(self._discover(account, credential)) + if not models: + raise CatalogHttpError("catalog_contains_no_models") + self.store.replace_catalog(account, models) + provider_rows[account.provider_account_id] = { + "status": "refreshed", + "model_count": len(models), + "error_code": None, + } + except CatalogHttpError as exc: + provider_rows[account.provider_account_id] = self._failed_refresh(account, exc.code) + except Exception: + provider_rows[account.provider_account_id] = self._failed_refresh( + account, "catalog_adapter_failure" + ) + candidates = self.store.enabled_models() + self.last_refresh_summary = { + "provider_accounts": provider_rows, + "candidate_model_count": len(candidates), + "measurement_status": "provider_catalog_snapshot", + } + if not candidates: + raise ProviderCatalogUnavailable("no usable provider model exists after catalog refresh") + return self.last_refresh_summary + + def _failed_refresh(self, account: ProviderAccount, code: str) -> dict[str, Any]: + """Record failure and classify whether last-known-good service remains available.""" + self.store.record_failure(account, code) + stale_available = self.store.has_models(account.provider_account_id) + return { + "status": "stale_available" if stale_available else "failed", + "model_count": 0, + "error_code": code, + } + + def candidate_agents(self) -> list[ModelAgent]: + """Convert enabled catalog rows into role-tagged, failover-capable agents.""" + agents: list[ModelAgent] = [] + for record in self.store.enabled_models(): + account = self._account_by_id[record.provider_account_id] + model = record.model + agents.append( + ModelAgent( + id=_agent_id(account.provider_account_id, model.model_name), + model=model.model_name, + base_url=account.base_url, + credential_key=account.credential_name, + tags=_agent_tags(model), + priority=account.priority_rank + _model_priority(model), + provider_name=account.provider_name, + ) + ) + return agents + + +def bootstrap_provider_credentials( + environment: Mapping[str, str], + *, + require_all: bool, + accounts: Sequence[ProviderAccount] = DEFAULT_PROVIDER_ACCOUNTS, +) -> dict[str, list[str]]: + """Transport the fixed provider-secret inventory into the credential registry. + + Validation happens before mutation when ``require_all`` is true, preventing a + partially updated production credential set. The returned summary contains + names only and is safe for CI logs. + """ + values = { + account.credential_name: str(environment.get(account.credential_name, "")).strip() + for account in accounts + } + missing = [name for name, value in values.items() if not value] + if require_all and missing: + raise ProviderCatalogUnavailable("provider credential inventory is incomplete") + registered: list[str] = [] + for account in accounts: + value = values[account.credential_name] + if value: + register_credential(account.credential_name, value) + registered.append(account.credential_name) + return {"registered_credentials": registered, "missing_credentials": missing} + + +def normalize_models_document(document: Mapping[str, Any]) -> list[DiscoveredModel]: + """Normalize common OpenAI/OpenRouter/Bytez listing shapes into model rows.""" + raw_rows: Any = document.get("data") + if not isinstance(raw_rows, list): + raw_rows = document.get("models") + if isinstance(raw_rows, Mapping): + raw_rows = list(raw_rows.values()) + if not isinstance(raw_rows, list): + return [] + models: dict[str, DiscoveredModel] = {} + for raw in raw_rows: + if isinstance(raw, str): + raw = {"id": raw} + if not isinstance(raw, Mapping): + continue + name = str(raw.get("id") or raw.get("model") or raw.get("name") or "").strip() + if not name or len(name) > 512: + continue + display_name = str(raw.get("name") or raw.get("display_name") or name).strip()[:512] or name + architecture = raw.get("architecture") if isinstance(raw.get("architecture"), Mapping) else {} + input_modalities = _string_values( + architecture.get("input_modalities") or raw.get("input_modalities") or raw.get("modalities") + ) + output_modalities = _string_values( + architecture.get("output_modalities") or raw.get("output_modalities") + ) + modalities = tuple(sorted(set(input_modalities + output_modalities) or {"text"})) + capabilities = _infer_capabilities(name, raw, modalities) + context_window = _optional_positive_int( + raw.get("context_length") or raw.get("context_window") or raw.get("max_context_length") + ) + pricing = raw.get("pricing") if isinstance(raw.get("pricing"), Mapping) else {} + input_price = _per_token_price_to_million( + pricing.get("prompt") or raw.get("input_price_per_token") + ) + output_price = _per_token_price_to_million( + pricing.get("completion") or raw.get("output_price_per_token") + ) + models[name] = DiscoveredModel( + model_name=name, + display_name=display_name, + capabilities=capabilities, + modalities=modalities, + context_window=context_window, + input_price_usd_per_million=input_price, + output_price_usd_per_million=output_price, + ) + return [models[name] for name in sorted(models)] + + +def build_catalog_orchestrator( + store: ProviderCatalogStore, + *, + accounts: Sequence[ProviderAccount] = DEFAULT_PROVIDER_ACCOUNTS, + client: ModelClient | None = None, + **orchestrator_options: Any, +) -> TaskOrchestrator: + """Build a normal :class:`TaskOrchestrator` from the durable candidate pool.""" + service = ProviderCatalogService(store=store, accounts=accounts) + agents = service.candidate_agents() + if not agents: + raise ProviderCatalogUnavailable("provider catalog contains no enabled agents") + return TaskOrchestrator( + agents, + client=client or ProviderAwareModelClient(), + **orchestrator_options, + ) + + +def _normalize_bytez_output(document: Mapping[str, Any]) -> str: + """Extract text from the bounded native Bytez response contract.""" + output = document.get("output") + if isinstance(output, str) and output: + return output + if isinstance(output, Mapping): + content = output.get("content") or output.get("text") + if isinstance(content, str) and content: + return content + raise ProviderCatalogUnavailable("Bytez response shape is unsupported") + + +def _secure_json_request( # pragma: no cover - real credentialed network boundary + *, + method: str, + url: str, + header_name: str, + authorization: str, + payload: Mapping[str, Any] | None, + timeout_seconds: float, + transient_status: Sequence[int], +) -> dict[str, Any]: + parsed = urlparse(url) + if parsed.scheme != "https" or not parsed.hostname: + raise CatalogHttpError("catalog_url_must_use_https") + if parsed.username is not None or parsed.password is not None: + raise CatalogHttpError("catalog_url_must_not_contain_userinfo") + port = parsed.port or 443 + addresses = _validated_global_addresses(parsed.hostname, port) + target = parsed.path or "/" + if parsed.query: + target = f"{target}?{parsed.query}" + body = None if payload is None else json.dumps(payload).encode("utf-8") + headers = { + header_name: authorization, + "Accept": "application/json", + "Connection": "close", + "User-Agent": "contextual-orchestrator-provider-catalog/1", + } + if body is not None: + headers["Content-Type"] = "application/json" + last_network_error: BaseException | None = None + for address in addresses: + connection = _PinnedCatalogConnection( + parsed.hostname, + address, + port, + timeout_seconds, + ssl.create_default_context(), + ) + try: + connection.request(method, target, body=body, headers=headers) + response = connection.getresponse() + status = response.status + if status >= 300: + response.close() + connection.close() + if status in {401, 403}: + raise CatalogHttpError("catalog_authentication_failed") + raise CatalogHttpError( + f"catalog_http_{status}", transient=status in transient_status + ) + content_type = (response.getheader("Content-Type") or "").lower() + if "json" not in content_type: + response.close() + connection.close() + raise CatalogHttpError("catalog_content_type_invalid") + raw_payload = response.read(CATALOG_RESPONSE_MAX_BYTES + 1) + response.close() + connection.close() + if len(raw_payload) > CATALOG_RESPONSE_MAX_BYTES: + raise CatalogHttpError("catalog_response_too_large") + try: + document = json.loads(raw_payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError, RecursionError): + raise CatalogHttpError("catalog_json_invalid") from None + if not isinstance(document, dict): + raise CatalogHttpError("catalog_json_must_be_object") + return document + except CatalogHttpError: + raise + except (OSError, http.client.HTTPException, TimeoutError) as exc: + connection.close() + last_network_error = exc + raise CatalogHttpError("catalog_network_failure", transient=True) from last_network_error + + +def _upsert_account_row(cursor: Any, account: ProviderAccount) -> None: # pragma: no cover - SQL adapter + """Execute the parameter-bound provider-account upsert.""" + cursor.execute( + "INSERT INTO provider_accounts (" + "provider_account_id, provider_name, credential_name, base_url, models_path, " + "transport_name, auth_header_name, auth_prefix, enabled_flag, priority_rank, updated_at) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, now()) " + "ON CONFLICT (provider_account_id) DO UPDATE SET " + "provider_name = EXCLUDED.provider_name, credential_name = EXCLUDED.credential_name, " + "base_url = EXCLUDED.base_url, models_path = EXCLUDED.models_path, " + "transport_name = EXCLUDED.transport_name, auth_header_name = EXCLUDED.auth_header_name, " + "auth_prefix = EXCLUDED.auth_prefix, enabled_flag = EXCLUDED.enabled_flag, " + "priority_rank = EXCLUDED.priority_rank, updated_at = now()", + ( + account.provider_account_id, + account.provider_name, + account.credential_name, + account.base_url, + account.models_path, + account.transport_name, + account.auth_header_name, + account.auth_prefix, + account.enabled, + account.priority_rank, + ), + ) + + +def _insert_refresh_row( # pragma: no cover - SQL adapter + cursor: Any, + account_id: str, + status: str, + count: int, + error_code: str | None, + started_at: datetime, + finished_at: datetime, +) -> None: + """Insert one immutable provider refresh evidence row.""" + cursor.execute( + "INSERT INTO catalog_refresh_runs (" + "catalog_refresh_id, provider_account_id, refresh_status, observed_model_count, " + "error_code, started_at, finished_at) VALUES (%s, %s, %s, %s, %s, %s, %s)", + ( + _refresh_id(account_id, finished_at.isoformat()), + account_id, + status, + count, + error_code, + started_at, + finished_at, + ), + ) + + +def _validated_global_addresses(hostname: str, port: int) -> tuple[str, ...]: # pragma: no cover - DNS boundary + """Resolve and accept only globally routable addresses for credentialed egress.""" + addresses: list[str] = [] + try: + candidates = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + except socket.gaierror: + raise CatalogHttpError("catalog_dns_failure", transient=True) from None + for candidate in candidates: + address = ipaddress.ip_address(candidate[4][0]) + if ( + not address.is_global + or address.is_private + or address.is_loopback + or address.is_link_local + or address.is_multicast + or address.is_reserved + ): + raise CatalogHttpError("catalog_destination_not_public") + value = str(address) + if value not in addresses: + addresses.append(value) + if not addresses: + raise CatalogHttpError("catalog_dns_empty", transient=True) + return tuple(addresses) + + +def _infer_capabilities( + model_name: str, + raw: Mapping[str, Any], + modalities: Sequence[str], +) -> tuple[str, ...]: + """Infer conservative routing tags from provider metadata and model naming.""" + lowered = model_name.lower() + capabilities = {value.lower() for value in _string_values(raw.get("capabilities"))} + if any(token in lowered for token in ("embed", "embedding")): + capabilities.add("embeddings") + elif "rerank" in lowered: + capabilities.add("reranking") + elif "moderation" in lowered: + capabilities.add("moderation") + else: + capabilities.add("chat") + if any(token in lowered for token in ("reason", "o1", "o3", "r1", "thinking")): + capabilities.add("reasoning") + if any(token in lowered for token in ("code", "coder", "codestral", "devstral")): + capabilities.add("coding") + if "image" in modalities or "vision" in lowered or "vl" in lowered: + capabilities.add("vision") + if "audio" in modalities or any(token in lowered for token in ("audio", "whisper", "speech")): + capabilities.add("audio") + if "guard" in lowered: + capabilities.add("moderation") + return tuple(sorted(capabilities)) + + +def _agent_tags(model: DiscoveredModel) -> tuple[str, ...]: + """Map provider capabilities into the orchestrator's role/domain tag vocabulary.""" + tags: set[str] = set(model.capabilities) + if "chat" in tags: + tags.update(("writing", "summarization", "classification")) + if "reasoning" in tags: + tags.update(("planning", "research", "verification")) + if "coding" in tags: + tags.update(("implementation", "debugging")) + if "vision" in tags: + tags.update(("image", "multimodal")) + if "audio" in tags: + tags.update(("speech", "multimodal")) + return tuple(sorted(tags)) + + +def _model_priority(model: DiscoveredModel) -> int: + """Use context and known price only as small ties after role/capability scoring.""" + score = min(3, (model.context_window or 0) // 100_000) + known_prices = [ + value + for value in ( + model.input_price_usd_per_million, + model.output_price_usd_per_million, + ) + if value is not None + ] + if known_prices: + average = sum(known_prices) / len(known_prices) + score += max(0, 2 - min(2, int(average))) + return score + + +def _agent_id(provider_account_id: str, model_name: str) -> str: + """Create a bounded two-or-more-word snake-case agent identifier.""" + slug = re.sub(r"[^a-z0-9]+", "_", model_name.lower()).strip("_") or "model_worker" + digest = hashlib.sha256(model_name.encode("utf-8")).hexdigest()[:8] + return f"{provider_account_id}_{slug}_{digest}"[:120].rstrip("_") + + +def _provider_model_id(provider_account_id: str, model_name: str) -> str: # pragma: no cover - SQL adapter + """Return a stable non-secret identifier for one account/model pair.""" + material = f"{provider_account_id}\0{model_name}".encode("utf-8") + return f"provider_model_{hashlib.sha256(material).hexdigest()}" + + +def _refresh_id(account_id: str, timestamp: str) -> str: + """Return an immutable refresh identifier without exposing credentials.""" + material = f"{account_id}\0{timestamp}".encode("utf-8") + return f"catalog_refresh_{hashlib.sha256(material).hexdigest()}" + + +def _string_values(value: Any) -> list[str]: + """Return bounded, non-empty strings from scalar or sequence metadata.""" + if isinstance(value, str): + values: Iterable[Any] = (value,) + elif isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + values = value + else: + return [] + result: list[str] = [] + for item in values: + if isinstance(item, str): + normalized = item.strip().lower() + if normalized and len(normalized) <= 128: + result.append(normalized) + return result + + +def _optional_positive_int(value: Any) -> int | None: + """Return a positive integer metadata value, rejecting booleans and overflow.""" + if isinstance(value, bool) or value is None: + return None + try: + parsed = int(value) + except (TypeError, ValueError, OverflowError): + return None + return parsed if 0 < parsed <= 10_000_000_000 else None + + +def _per_token_price_to_million(value: Any) -> float | None: + """Convert a finite non-negative per-token USD price to per-million units.""" + if isinstance(value, bool) or value is None: + return None + try: + parsed = float(value) + except (TypeError, ValueError, OverflowError): + return None + if not math.isfinite(parsed) or parsed < 0: + return None + return parsed * 1_000_000 + + +def _optional_float(value: Any) -> float | None: # pragma: no cover - SQL adapter + """Convert a finite database numeric value to float, preserving null.""" + if value is None: + return None + parsed = float(value) + return parsed if math.isfinite(parsed) else None + + +def _utc_now() -> datetime: + """Return the current timezone-aware UTC timestamp.""" + return datetime.now(timezone.utc) + + +def _utc_now_text() -> str: + """Return the current UTC timestamp as an ISO-8601 string.""" + return _utc_now().isoformat() + + +def _safe_cli_summary( # pragma: no cover - CLI integration + credential_summary: Mapping[str, Any], catalog_summary: Mapping[str, Any] +) -> dict[str, Any]: + """Build a log-safe bootstrap summary containing no credential values.""" + return { + "registered_credentials": list(credential_summary.get("registered_credentials", [])), + "missing_credentials": list(credential_summary.get("missing_credentials", [])), + "candidate_model_count": int(catalog_summary.get("candidate_model_count", 0)), + "provider_accounts": dict(catalog_summary.get("provider_accounts", {})), + "measurement_status": "provider_catalog_bootstrap", + } + + +def _write_agents_file(path: str, agents: Sequence[ModelAgent]) -> None: # pragma: no cover - CLI integration + """Atomically write a secret-free agent configuration JSON document.""" + target = os.path.abspath(path) + os.makedirs(os.path.dirname(target) or ".", exist_ok=True) + temporary = f"{target}.tmp-{os.getpid()}" + try: + with open(temporary, "w", encoding="utf-8") as handle: + json.dump( + {"agents": [agent.to_config() for agent in agents]}, + handle, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + handle.write("\n") + os.replace(temporary, target) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def main(argv: Sequence[str] | None = None) -> int: # pragma: no cover - CLI integration + """Bootstrap credentials, refresh the durable catalog, and optionally export agents.""" + parser = argparse.ArgumentParser(description="Bootstrap and refresh the durable provider catalog.") + parser.add_argument("command", choices=("bootstrap-and-sync", "sync", "export-agents")) + parser.add_argument( + "--catalog-dsn", + default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN") + or os.environ.get("CONTEXTUAL_ORCHESTRATOR_KV_DSN", ""), + help="PostgreSQL DSN used for provider metadata (bootstrap transport only).", + ) + parser.add_argument("--require-all", action="store_true") + parser.add_argument("--agents-output", default="") + args = parser.parse_args(list(argv) if argv is not None else None) + + store = PostgresProviderCatalogStore(args.catalog_dsn) + credential_summary: dict[str, list[str]] = { + "registered_credentials": [], + "missing_credentials": [], + } + if args.command == "bootstrap-and-sync": + credential_summary = bootstrap_provider_credentials(os.environ, require_all=args.require_all) + service = ProviderCatalogService(store=store) + if args.command in {"bootstrap-and-sync", "sync"}: + catalog_summary = service.refresh_all() + else: + catalog_summary = { + "candidate_model_count": len(store.enabled_models()), + "provider_accounts": {}, + } + agents = service.candidate_agents() + if not agents: + raise ProviderCatalogUnavailable("provider catalog contains no enabled agents") + if args.agents_output: + _write_agents_file(args.agents_output, agents) + print(json.dumps(_safe_cli_summary(credential_summary, catalog_summary), sort_keys=True)) + return 0 + + +if __name__ == "__main__": # pragma: no cover - CLI integration + try: + raise SystemExit(main()) + except ProviderCatalogUnavailable as exc: + print( + json.dumps({"error": "provider_catalog_unavailable", "message": str(exc)}), + file=sys.stderr, + ) + raise SystemExit(2) from None diff --git a/contextual_orchestrator/provider_transport.py b/contextual_orchestrator/provider_transport.py new file mode 100644 index 000000000..3e1b76bd2 --- /dev/null +++ b/contextual_orchestrator/provider_transport.py @@ -0,0 +1,442 @@ +"""DNS-pinned HTTPS primitives for validated model-provider egress. + +``ModelClient`` owns policy validation and request dispatch directly. This +module contains only focused connection, response-cleanup, bounded-consumption, +strict provider-response decoding, and public-address validation helpers, so +importing the package never mutates another class. +""" + +from __future__ import annotations + +from contextlib import suppress +import http.client +import ipaddress +import json +import math +import socket +import ssl +from typing import Any, Iterator + +from .credentials import NotConfigured + + +PROVIDER_RESPONSE_MAX_BYTES = 8 * 1024 * 1024 +"""Maximum bytes consumed from one untrusted provider HTTP response.""" + + +def _reject_non_finite_json_constant(_value: str) -> None: + """Reject Python JSON extensions that RFC 8259 does not permit.""" + raise ValueError("non-finite JSON number") + + +def _parse_finite_json_float(value: str) -> float: + """Decode one JSON float while rejecting overflow into a non-finite value.""" + parsed = float(value) + if not math.isfinite(parsed): + raise ValueError("non-finite JSON number") + return parsed + + +def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Build one JSON object while rejecting ambiguous duplicate member names.""" + result: dict[str, Any] = {} + for name, value in pairs: + if name in result: + raise ValueError("duplicate JSON member name") + result[name] = value + return result + + +def _parse_provider_json_object_text(text: str) -> dict[str, Any]: + """Parse strict RFC 8259 JSON text and require one top-level object. + + Python's default decoder intentionally accepts ``NaN`` and infinities and + silently keeps the final value of duplicate object members. Provider + responses cross a trust boundary, so those extensions and finite-syntax + numbers that overflow Python's runtime float representation are rejected + before orchestration code can interpret ambiguous data. Decoder exceptions + are replaced without chaining because ``JSONDecodeError`` retains the entire + untrusted document in its ``doc`` attribute. + """ + try: + value = json.loads( + text, + object_pairs_hook=_unique_json_object, + parse_constant=_reject_non_finite_json_constant, + parse_float=_parse_finite_json_float, + ) + except (ValueError, RecursionError): + raise RuntimeError("provider JSON response is malformed") from None + if not isinstance(value, dict): + raise RuntimeError("provider JSON response must be an object") + return value + + +def _decode_provider_json_object(payload: bytes) -> dict[str, Any]: + """Decode one strict UTF-8 provider JSON object with redacted failures.""" + try: + text = payload.decode("utf-8") + except UnicodeDecodeError: + raise RuntimeError("provider JSON response is malformed") from None + return _parse_provider_json_object_text(text) + + +def _encode_provider_json_object(payload: bytes) -> bytes: + """Return canonical UTF-8 bytes after strict provider-object validation.""" + value = _decode_provider_json_object(payload) + try: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + ).encode("utf-8") + except (TypeError, ValueError, RecursionError): + raise RuntimeError("provider JSON response is malformed") from None + + +def _encode_provider_json_lines(payload: bytes) -> bytes: + """Validate and canonicalize an OpenAI Batch API JSON Lines response. + + Every non-empty line must independently be a strict UTF-8 JSON object. The + returned bytes remain line-addressable for the existing batch parser, but a + malformed provider document is rejected here before a later ``json.loads`` + exception can retain private response content. + """ + try: + text = payload.decode("utf-8") + except UnicodeDecodeError: + raise RuntimeError("provider JSON Lines response is malformed") from None + normalized_lines: list[str] = [] + try: + for line in text.splitlines(): + if not line.strip(): + continue + value = json.loads( + line, + object_pairs_hook=_unique_json_object, + parse_constant=_reject_non_finite_json_constant, + parse_float=_parse_finite_json_float, + ) + if not isinstance(value, dict): + raise ValueError("JSON Lines row must be an object") + normalized_lines.append( + json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + ) + ) + except (TypeError, ValueError, RecursionError): + raise RuntimeError("provider JSON Lines response is malformed") from None + if not normalized_lines: + raise RuntimeError("provider JSON Lines response is malformed") + return "\n".join(normalized_lines).encode("utf-8") + + +def _is_batch_output_content_path(request_path: str) -> bool: + """Return whether a validated target is a provider file-content endpoint.""" + path = request_path.partition("?")[0] + segments = [segment for segment in path.split("/") if segment] + return ( + len(segments) >= 3 + and segments[-3] == "files" + and segments[-1] == "content" + ) + + +class _PinnedHTTPSConnection(http.client.HTTPSConnection): + """Connect to one validated IP while retaining the provider hostname for TLS.""" + + def __init__( + self, + server_hostname: str, + pinned_ip: str, + port: int, + timeout: float, + context: ssl.SSLContext, + ) -> None: + """Configure a direct TLS connection to a previously validated address.""" + super().__init__(server_hostname, port=port, timeout=timeout, context=context) + self._pinned_ip = pinned_ip + self._server_hostname = server_hostname + + def request( + self, + method: str, + url: str, + body: Any = None, + headers: dict[str, str] | None = None, + *, + encode_chunked: bool = False, + ) -> None: + """Require a current non-empty Bearer credential before any socket can open. + + Provider credentials are resolved immediately before request construction. + A credential can still be revoked between DNS validation and dispatch, in + which case ``ModelClient`` produces an empty Bearer value. This last + pre-socket boundary therefore rejects missing or empty authorization so a + revoked secret can never degrade into unauthenticated provider egress. + + The exact request target is also retained on this connection. The paired + response wrapper uses that already-validated target to distinguish normal + JSON objects from the Batch API's JSON Lines file-content response without + asking orchestration call sites to duplicate transport trust policy. + """ + request_headers = headers or {} + authorization = next( + ( + str(value) + for name, value in request_headers.items() + if name.lower() == "authorization" + ), + "", + ) + scheme, separator, credential = authorization.partition(" ") + if scheme.lower() != "bearer" or not separator or not credential.strip(): + self.close() + raise NotConfigured( + "provider HTTPS egress requires a current non-empty Bearer credential" + ) + self._provider_request_path = url + super().request( + method, + url, + body=body, + headers=request_headers, + encode_chunked=encode_chunked, + ) + + def connect(self) -> None: + """Dial the pinned IP and verify the certificate against the original host.""" + raw_socket = socket.create_connection( + (self._pinned_ip, self.port), + self.timeout, + self.source_address, + ) + try: + self.sock = self._context.wrap_socket( + raw_socket, + server_hostname=self._server_hostname, + ) + except Exception: # noqa: BLE001 - close the raw socket, then preserve the TLS failure. + raw_socket.close() + raise + + +def _content_length_exceeds_budget(value: str, max_bytes: int) -> bool: + """Validate one Content-Length field and compare without integer overflow.""" + canonical_members: list[str] = [] + for member in value.split(","): + token = member.strip(" \t") + if not token or not token.isascii() or not token.isdigit(): + raise ValueError("invalid Content-Length") + canonical_members.append(token.lstrip("0") or "0") + declared = canonical_members[0] + if any(member != declared for member in canonical_members[1:]): + raise ValueError("conflicting Content-Length") + limit = str(max_bytes) + if len(declared) > len(limit): + return True + if len(declared) < len(limit): + return False + return declared > limit + + +class _ProviderHTTPResponse: + """Bound provider bytes and deterministically close response resources.""" + + def __init__( + self, + response: Any, + connection: Any, + max_bytes: int = PROVIDER_RESPONSE_MAX_BYTES, + ) -> None: + """Retain resources and initialize one cumulative response-byte budget.""" + if isinstance(max_bytes, bool) or not isinstance(max_bytes, int) or max_bytes <= 0: + raise ValueError("provider response byte limit must be a positive integer") + self._response = response + self._connection = connection + self._max_bytes = max_bytes + self._bytes_read = 0 + try: + self._validate_response_framing() + except Exception: + with suppress(Exception): + self.close() + raise + + def _validate_response_framing(self) -> None: + """Reject malformed, ambiguous, or already over-budget HTTP framing.""" + if not isinstance(self._response, http.client.HTTPResponse): + return + try: + content_length = self._response.getheader("Content-Length") + transfer_encoding = self._response.getheader("Transfer-Encoding") + except Exception: + raise RuntimeError( + "provider response headers could not be validated" + ) from None + if content_length is not None and transfer_encoding is not None: + raise RuntimeError("provider response framing is ambiguous") + if transfer_encoding is not None: + if transfer_encoding.lower() != "chunked": + raise RuntimeError("provider response transfer encoding is unsupported") + return + if content_length is None: + return + try: + exceeds_budget = _content_length_exceeds_budget( + content_length, + self._max_bytes, + ) + except ValueError: + raise RuntimeError( + "provider response content length is invalid" + ) from None + if exceeds_budget: + raise RuntimeError("provider response byte limit exceeded") + + def __enter__(self) -> "_ProviderHTTPResponse": + """Return this response wrapper from a context manager.""" + return self + + def __exit__(self, _exc_type: Any, _exc: Any, _traceback: Any) -> None: + """Close the response and its connection when leaving the context.""" + self.close() + + def __iter__(self) -> Iterator[bytes]: + """Yield only bounded, valid server-sent-event response lines. + + Real ``HTTPResponse`` iteration is reserved for provider streaming. It + therefore requires the standardized ``text/event-stream`` media type + before consuming any body bytes, then uses size-limited ``readline`` + calls so one pathological line cannot allocate beyond the remaining + budget before inspection. Every ``data:`` frame must contain a strict + JSON object until the OpenAI-compatible terminal ``[DONE]`` marker + arrives. A missing or incorrect media type, malformed data, or end-of-file + before that marker fails closed instead of turning a non-stream or partial + model answer into successful orchestration output. Lightweight non-HTTP + test doubles retain ordinary iteration while still receiving cumulative + byte accounting. + """ + if isinstance(self._response, http.client.HTTPResponse): + try: + content_type = self._response.getheader("Content-Type", "") + except Exception: + raise RuntimeError( + "provider stream content type could not be validated" + ) from None + media_type = content_type.partition(";")[0].strip().lower() + if media_type != "text/event-stream": + raise RuntimeError( + "provider stream requires text/event-stream content type" + ) + while True: + line = self._response.readline(self._remaining_bytes() + 1) + if not line: + raise RuntimeError("provider stream terminated before [DONE]") + bounded_line = self._account(line) + try: + text = bounded_line.decode("utf-8").strip() + except UnicodeDecodeError: + raise RuntimeError("malformed provider stream event") from None + if text.startswith("data:"): + data = text[len("data:") :].strip() + if data == "[DONE]": + return + try: + _parse_provider_json_object_text(data) + except RuntimeError: + raise RuntimeError( + "malformed provider stream event" + ) from None + yield bounded_line + else: + for line in self._response: + yield self._account(line) + + def __getattr__(self, name: str) -> Any: + """Delegate response metadata such as status and headers.""" + return getattr(self._response, name) + + def _remaining_bytes(self) -> int: + """Return bytes still available before the response must fail closed.""" + return self._max_bytes - self._bytes_read + + def _account(self, chunk: bytes) -> bytes: + """Charge one consumed chunk to the cumulative response-byte budget.""" + next_total = self._bytes_read + len(chunk) + if next_total > self._max_bytes: + raise RuntimeError("provider response byte limit exceeded") + self._bytes_read = next_total + return chunk + + def _read_bounded_bytes(self, amt: int | None = None) -> bytes: + """Consume no more than the remaining response-byte budget.""" + remaining = self._remaining_bytes() + if amt is None or amt < 0 or amt > remaining: + requested = remaining + 1 + else: + requested = amt + return self._account(self._response.read(requested)) + + def read_json_object(self) -> dict[str, Any]: + """Read one bounded provider body as a strict UTF-8 JSON object. + + This explicit method is useful to focused transport callers and tests. + Normal validated HTTPS model-client reads receive the same protection + automatically through ``read`` using the request path captured by + ``_PinnedHTTPSConnection``. + """ + return _decode_provider_json_object(self._read_bounded_bytes()) + + def read(self, amt: int | None = None) -> bytes: + """Read bounded bytes and validate complete validated-provider documents. + + Explicit partial reads remain byte-oriented. A complete response from a + DNS-pinned HTTPS request is normalized only after strict validation: Batch + file-content endpoints are JSON Lines, while all other non-stream provider + endpoints used by ``ModelClient`` return one JSON object. Connections that + do not carry a validated request target (lightweight tests and the explicit + loopback integration seam) preserve the historical bounded-byte behavior. + """ + payload = self._read_bounded_bytes(amt) + request_path = getattr(self._connection, "_provider_request_path", None) + if not isinstance(request_path, str) or not request_path: + return payload + if amt is not None and amt >= 0: + return payload + if _is_batch_output_content_path(request_path): + return _encode_provider_json_lines(payload) + return _encode_provider_json_object(payload) + + def close(self) -> None: + """Close both resources even when response cleanup raises.""" + try: + self._response.close() + finally: + self._connection.close() + + +def _validated_public_addresses(hostname: str, port: int, provider_label: str) -> tuple[str, ...]: + """Resolve, validate, and deduplicate addresses approved for one connection.""" + validated_addresses: list[str] = [] + for address in socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM): + resolved_address = ipaddress.ip_address(address[4][0]) + if ( + not resolved_address.is_global + or resolved_address.is_private + or resolved_address.is_loopback + or resolved_address.is_link_local + or resolved_address.is_multicast + or resolved_address.is_reserved + ): + raise RuntimeError(f"{provider_label} provider resolves to non-public address") + normalized_address = str(resolved_address) + if normalized_address not in validated_addresses: + validated_addresses.append(normalized_address) + if not validated_addresses: + raise RuntimeError(f"{provider_label} provider host did not resolve") + return tuple(validated_addresses) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index c58d4cb79..9b15e2b73 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -327,7 +327,10 @@ def build_server( clearfolio_url = clearfolio_url.rstrip("/") class Handler(BaseHTTPRequestHandler): + """Handle one authenticated HTTP request against the supplied orchestrator.""" + def do_GET(self) -> None: # noqa: N802 + """Return health, admin, evidence, workflow, and batch resources.""" parsed = urllib.parse.urlparse(self.path) path = parsed.path query = urllib.parse.parse_qs(parsed.query) @@ -654,6 +657,7 @@ def do_GET(self) -> None: # noqa: N802 self._send_error(500, "internal_error", "internal server error") def do_PATCH(self) -> None: # noqa: N802 + """Apply validated operator changes to one worker-agent resource.""" try: self._authorize("admin") path = urllib.parse.urlparse(self.path).path @@ -677,6 +681,7 @@ def do_PATCH(self) -> None: # noqa: N802 self._send_error(500, "internal_error", "internal server error") def do_DELETE(self) -> None: # noqa: N802 + """Remove one worker agent through the authenticated admin surface.""" try: self._authorize("admin") path = urllib.parse.urlparse(self.path).path @@ -697,6 +702,7 @@ def do_DELETE(self) -> None: # noqa: N802 self._send_error(500, "internal_error", "internal server error") def do_POST(self) -> None: # noqa: N802 + """Create agents or execute validated inference and workflow requests.""" try: path = urllib.parse.urlparse(self.path).path scope = "admin" if path == "/admin/simulate" or path.startswith("/api/v1/agent_pools/") else "inference" @@ -967,6 +973,7 @@ def _read_json(self) -> dict[str, Any]: return _coerce_json(raw) if raw else {} def log_message(self, format: str, *args: object) -> None: + """Suppress the base server's unstructured request logging.""" return def _send_error( diff --git a/docs/database_design.sql b/docs/database_design.sql index fb5892479..90ab301fb 100644 --- a/docs/database_design.sql +++ b/docs/database_design.sql @@ -10,6 +10,63 @@ create table agent_pool ( updated_at timestamptz not null default now() ); +create table provider_accounts ( + provider_account_id text primary key, + provider_name text not null, + credential_name text not null, + base_url text not null, + models_path text, + transport_name text not null, + auth_header_name text not null, + auth_prefix text not null, + enabled_flag boolean not null default true, + priority_rank integer not null default 0, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +create table provider_models ( + provider_model_id text primary key, + provider_account_id text not null references provider_accounts(provider_account_id), + model_name text not null, + display_name text not null, + context_window integer, + input_price_usd_per_million numeric(20, 8), + output_price_usd_per_million numeric(20, 8), + enabled_flag boolean not null default true, + first_discovered_at timestamptz not null, + last_seen_at timestamptz not null, + unique (provider_account_id, model_name) +); + +create table model_capabilities ( + provider_model_id text not null references provider_models(provider_model_id) on delete cascade, + capability_name text not null, + primary key (provider_model_id, capability_name) +); + +create table model_modalities ( + provider_model_id text not null references provider_models(provider_model_id) on delete cascade, + modality_name text not null, + primary key (provider_model_id, modality_name) +); + +create table catalog_refresh_runs ( + catalog_refresh_id text primary key, + provider_account_id text not null references provider_accounts(provider_account_id), + refresh_status text not null, + observed_model_count integer not null default 0, + error_code text, + started_at timestamptz not null, + finished_at timestamptz not null +); + +create index provider_models_account_idx + on provider_models (provider_account_id, enabled_flag); + +create index catalog_refresh_account_idx + on catalog_refresh_runs (provider_account_id, finished_at desc); + create table orchestration_policy ( policy_id text primary key, policy_name text not null, diff --git a/docs/doctoring/atheris-interpreter-lock.md b/docs/doctoring/atheris-interpreter-lock.md new file mode 100644 index 000000000..ae79b8f8f --- /dev/null +++ b/docs/doctoring/atheris-interpreter-lock.md @@ -0,0 +1,67 @@ +# Atheris interpreter lock evidence + +## Decision record + +Contextual Orchestrator uses one universal, hash-locked fuzz dependency file across validation environments. The repository fuzz runner uses CPython 3.11, while central coverage evidence can run on CPython 3.13 or later. The lock therefore partitions Atheris releases with standardized Python environment markers rather than maintaining divergent unreviewed lock files. + +## Standards basis + +The Python Packaging dependency-specifier specification defines environment markers as conditional dependency rules evaluated for the active installation environment. `python_version` and `python_full_version` are version-typed marker fields, and ordered comparisons such as `<` and `>=` use version-specifier semantics. Mutually exclusive markers are therefore the portable standards-based mechanism for selecting one interpreter-compatible dependency release. + +The project metadata uses: + +```text +atheris==3.0.0; python_version < "3.13" +atheris==3.1.0; python_version >= "3.13" +``` + +The generated universal requirements lock expresses the equivalent boundary with `python_full_version`, preserving installation-tool compatibility while selecting exactly one Atheris release. + +## Artifact evidence + +PyPI records Atheris 3.0.0 artifacts for CPython 3.11 through 3.13 and Atheris 3.1.0 artifacts for CPython 3.12 through 3.14. The repository retains 3.0.0 for its established Python 3.11 fuzz runner and selects the newer 3.1.0 release for Python 3.13 and later coverage environments. + +The lock includes the published SHA-256 values used by the supported manylinux wheels and source distribution evidence: + +### Atheris 3.0.0 + +- `1f0929c7bc3040f3fe4102e557718734190cf2d7718bbb8e3ce6d3eb56ef5bb3` +- `510e502c57b6dc615fb174066407af620d4c7f73cf08a782c86e7761bf12c4eb` +- `8a5c8a781467c187da40fd29139784193e2647058831f837f675d0bb8cbd8746` +- `a402cdca8a650d1371050b1f9552eb4cdc488d2db64950d603c4560318365eac` + +### Atheris 3.1.0 + +- `315a0b5c819852b1ffe1ca72efc389c7724881f2c33e4aacb8c6bcec49bd5011` +- `ec5e11f21a4c197fe91f7aea2b2de88e623c73a21fc07b105ac6329a1588457b` +- `f8a9f51ce8369026e8eb7b7174835e8c4c85a1a6db5d9add36c15100779d2a39` + +Installation continues to use `--require-hashes`; no unhashed or network-selected fallback is introduced. + +## Verification contract + +`tests/test_fuzz_dependency_lock.py` treats project metadata and the universal lock as evidence. It evaluates representative Python 3.11, 3.13, and 3.14 environments and fails when requirements overlap, leave an interpreter uncovered, disagree between metadata and lock, or omit published SHA-256 evidence. + +The test does not import Atheris or use provider egress. It therefore remains deterministic and can run before the platform-specific wheel is installed. + +## CI trust boundary + +The generic repository coverage verifier and the native fuzz runner have different responsibilities. Generic coverage must materialize the exact current lock identity or report a blocker; it must never accept an older lock-key artifact or silently fall back to an unhashed installation. Native Atheris execution remains isolated in the dedicated fuzz workflow so a platform-specific fuzz engine cannot make ordinary statement, branch, docstring, or package evidence non-portable. + +When a centrally maintained reusable workflow is referenced by a mutable branch and its verifier is repaired, retry semantics matter. GitHub documents that re-running only failed jobs retains the called reusable workflow commit from the first attempt, while re-running all jobs resolves the workflow from the specified branch reference. Operators must therefore use a fresh pull-request event or an all-jobs rerun when validating a central workflow repair, and must record the resulting current-head workflow identity. An older failed-job retry is not evidence for the repaired verifier. + +## Applicability and uncertainty + +This record proves dependency-selection and artifact-integrity consistency for the repository's declared interpreter partition. It does not claim that every operating system or architecture has an Atheris wheel. Runners outside the documented Linux/CPython environments must perform their own artifact-availability review and must not remove `--require-hashes` to force installation. + +Artifact availability and hashes are time-sensitive upstream facts. They were rechecked on August 5, 2026. Future version changes require a new lock regeneration, focused contract update, and renewed evidence review. + +## APA 7 references + +GitHub. (2026). *Reusing workflow configurations*. GitHub Docs. https://docs.github.com/en/actions/reference/workflows-and-actions/reusing-workflow-configurations + +Google. (2025). *Atheris* (Version 3.0.0) [Computer software]. Python Package Index. https://pypi.org/project/atheris/3.0.0/ + +Google. (2026). *Atheris* (Version 3.1.0) [Computer software]. Python Package Index. https://pypi.org/project/atheris/3.1.0/ + +Python Packaging Authority. (2026). *Dependency specifiers*. Python Packaging User Guide. https://packaging.python.org/en/latest/specifications/dependency-specifiers/ diff --git a/docs/doctoring/durable-provider-catalog.md b/docs/doctoring/durable-provider-catalog.md new file mode 100644 index 000000000..3ffa92b14 --- /dev/null +++ b/docs/doctoring/durable-provider-catalog.md @@ -0,0 +1,221 @@ +# Durable Provider Catalog Doctoring + +## Purpose + +This record explains why provider credentials and model catalogs are separate, +how the five configured provider accounts become an orchestration pool, which +failures are tolerated, and which failures stop service. It is the operational +source for incident response, rollback, and audit review. + +## Invariants + +1. Provider API-key values exist only in the encrypted credential registry. +2. The provider catalog contains credential names, never secret values. +3. `NVIDIA_NIM_API_KEY` and `NVIDIA_NIM_API_KEY_SUB` are independent accounts. +4. Pull-request code never receives production provider or database secrets. +5. A configured PostgreSQL catalog/KV is authoritative; failure cannot silently + downgrade to process memory. +6. A failed provider refresh cannot disable its last-known-good models. +7. A complete successful refresh may disable models absent from that account's + new complete listing. +8. Zero usable candidates is a startup/sync failure, not permission to use mocks. +9. Capability and role fit outrank context and cost; price is a bounded tie-break. +10. Native Bytez requests use its Key/input contract; unsupported OpenAI + passthrough shapes fail closed. + +## Bootstrap sequence + +The trusted protected-default-branch workflow performs these actions: + +1. Require non-empty `CONTEXTUAL_ORCHESTRATOR_KV_DSN`, + `CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE`, and all five provider keys. +2. Add values to GitHub Actions masking without echoing them. +3. Select `CONTEXTUAL_ORCHESTRATOR_KV_BACKEND=postgres`. +4. Validate the complete fixed inventory before any provider credential write. +5. Upsert credentials through `register_credential()` into pgcrypto storage. +6. Refresh each provider account independently over bounded credentialed HTTPS. +7. Upsert normalized provider/model/capability/modality rows transactionally. +8. Preserve prior rows for failed accounts and classify them `stale_available`. +9. Generate a secret-free agent pool and reject zero candidates. +10. Inspect the safe summary and generated JSON for any exact secret value. + +Do not copy a provider key into `--agents`, repository variables, command-line +arguments, artifacts, cache keys, logs, issue comments, or deployment manifests. + +## Runtime sequence + +Start the gateway with the durable catalog connection: + +```bash +python -m contextual_orchestrator --serve \ + --provider-catalog-dsn "$CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN" \ + --admin-token "$CONTEXTUAL_ORCHESTRATOR_ADMIN_TOKEN" \ + --inference-token "$CONTEXTUAL_ORCHESTRATOR_INFERENCE_TOKEN" +``` + +The DSN connects to the catalog; it is not a provider API key. Startup loads only +enabled account/model rows. Each `ModelAgent` carries a credential name, and the +provider client resolves the current value from the credential registry at the +request boundary. Credential rotation therefore does not require rewriting +model rows. + +The existing orchestration engine receives the complete pool. Fast route mode +selects one model. Conduct mode selects role-appropriate Thinker, Worker, +Verifier, and Synthesizer candidates and retains other eligible accounts as +failover. Provider retries remain bounded and circuit breakers prevent a +persistently failing account from being selected continuously. + +## Exception matrix + +| Failure | Retry | Catalog mutation | Service effect | Required action | +| --- | --- | --- | --- | --- | +| DNS, connect, timeout | Bounded jitter | Failure row only | Stale models continue if present | Check egress/DNS/provider status | +| HTTP 408/409/425/429/5xx | Bounded jitter | Failure row only after exhaustion | Account stale/failed; peers continue | Inspect rate limits and provider SLO | +| HTTP 401/403 | No retry storm | Failure row only | Account stale/failed | Rotate or reauthorize named key | +| Redirect | Reject | Failure row only | Account stale/failed | Correct canonical endpoint; do not follow credential redirects | +| Private/reserved destination | Reject before credential send | Failure row only | Account stale/failed | Treat as SSRF/configuration incident | +| Non-JSON, duplicate/invalid JSON, excessive body | Reject | Failure row only | Account stale/failed | Treat as provider contract/security incident | +| Missing key in required bootstrap | No writes | None | Whole production bootstrap blocked | Configure the exact Actions secret | +| Missing key in optional local bootstrap | No write for account | Failure row on sync | Peers may continue | Seed key before production | +| PostgreSQL unavailable | No memory fallback | None | Sync/startup blocked | Restore authoritative database | +| Empty successful listing | No destructive replacement | Failure row only | Prior models stay; otherwise account failed | Verify provider list entitlement/contract | +| Bytez unsupported output | No repair/guess | Runtime failure only | Orchestrator may use another eligible account | Select supported native model/adapter | +| Bytez tool/Responses passthrough | Reject | None | Request fails closed | Route that contract to an OpenAI-compatible candidate | +| All accounts unavailable, no prior model | Bounded account attempts | Failure evidence where DB works | Gateway does not start | Restore at least one validated provider | + +Raw exception messages and response bodies are not public error contracts because +they can contain provider-controlled or sensitive content. Stable reason codes +are the operational interface. + +## Rotation procedure + +1. Add the new key value to the existing Actions secret name. +2. Manually run **Provider Catalog Sync** on protected `main` in the production + environment. +3. Confirm the safe summary reports the credential name and at least one model. +4. Confirm no account unexpectedly changed to `failed` or `stale_available`. +5. Send a bounded canary inference through that account. +6. Revoke the old provider key only after the canary succeeds. +7. Confirm the next scheduled refresh and runtime call resolve the new value. + +The database upsert replaces the encrypted value under the same credential name; +model rows and consuming services require no secret-bearing change. + +## Incident response + +### Suspected credential disclosure + +- Revoke/rotate the provider key immediately. +- Run trusted bootstrap to replace the encrypted registry value. +- Inspect Actions, application, proxy, database-audit, and provider logs for the + credential name and access time; do not paste the value into searches or tickets. +- Verify generated agent JSON and workflow summaries remain value-free. +- Treat a provider-side unauthorized model invocation as a security incident. + +### Catalog poisoning or malformed listing + +- Disable the affected `provider_accounts.enabled_flag` row. +- Preserve the response only in an access-controlled incident store; do not add + it to public CI logs. +- Confirm other providers still supply role coverage. +- Reproduce with a sanitized fixture and add a failing parser/transport test. +- Re-enable only after exact-head security tests and a clean refresh. + +### Database outage + +- Do not select memory mode as an automatic recovery mechanism. +- Restore the authoritative PostgreSQL service, network path, and pgcrypto + passphrase access. +- Validate `provider_credentials`, `provider_accounts`, `provider_models`, and the + latest `catalog_refresh_runs` before restarting the gateway. +- If emergency local mock service is intentionally required, start it as an + explicitly separate non-production deployment and label all evidence accordingly. + +## Rollback + +Code rollback may remove `--provider-catalog-dsn` and return a deployment to an +explicit reviewed agents file, but it must not copy API-key values into that +file or reintroduce runtime environment lookup. Keep the credential registry and +catalog tables during rollback; they are backward-compatible control-plane data +and preserve audit evidence. + +A schema rollback is normally unnecessary. If required, first export account, +model, capability, modality, and refresh metadata without secret values. Drop +catalog tables only after all catalog-backed services are stopped. Do not drop +`provider_credentials` as part of a model-catalog rollback. + +## Verification commands + +```bash +python -m pytest \ + tests/test_provider_catalog.py \ + tests/test_provider_catalog_coverage.py \ + tests/test_provider_catalog_cli.py -q +python -m coverage erase +python -m coverage run --branch -m pytest -q +python -m coverage report --fail-under=100 +interrogate --fail-under 100 contextual_orchestrator +python -m compileall -q contextual_orchestrator +python -m pip check +git diff --check +``` + +The trusted live sync is separate evidence. Pull-request success proves parser, +store, routing, failure, and workflow contracts without proving that any current +provider credential or production database is healthy. + +## Evidence interpretation + +- `refreshed`: current provider listing committed successfully. +- `stale_available`: current refresh failed, but prior enabled models remain. +- `failed`: refresh failed and that account has no usable prior model. +- `disabled`: governance deliberately excluded the account. +- `candidate_model_count`: enabled account/model pairs, not a quality claim. +- inferred capability: routing hint derived conservatively from metadata/name, + not a provider guarantee or benchmark result. +- price: stored only when supplied and finite; absent is `NULL`, not zero. + +## Research and standards rationale + +FrugalGPT and RouteLLM show that model selection can improve the +quality–cost frontier, but only when routing respects task quality rather than +using price alone. Fugu, TRINITY, and Conductor motivate a swappable model pool, +role assignment, selective context, and a route-versus-deep-orchestration split. +The durable catalog supplies that pool while retaining an auditable deterministic +policy until learned routing has a valid evaluation set. + +RFC 9110 informs retry and status classification: safe catalog GET operations may +be retried within explicit limits, while authentication failures and ambiguous +contracts fail fast. NIST AI RMF supports traceable inventory, monitoring, and +risk treatment. PostgreSQL pgcrypto provides the existing encryption-at-rest +boundary, while table separation prevents model metadata queries from exposing +secret values. + +## References + +Chen, L., Zaharia, M., & Zou, J. (2023). *FrugalGPT: How to use large language +models while reducing cost and improving performance*. arXiv. +https://doi.org/10.48550/arXiv.2305.05176 + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). +Internet Engineering Task Force. https://doi.org/10.17487/RFC9110 + +*Learning to orchestrate agents in natural language with the Conductor*. +(2025). arXiv. https://arxiv.org/abs/2512.04388 + +National Institute of Standards and Technology. (2023). *Artificial intelligence +risk management framework (AI RMF 1.0)* (NIST AI 100-1). +https://doi.org/10.6028/NIST.AI.100-1 + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., Kadous, +M. W., & Stoica, I. (2024). *RouteLLM: Learning to route LLMs with preference +data*. arXiv. https://doi.org/10.48550/arXiv.2406.18665 + +PostgreSQL Global Development Group. (2026). *pgcrypto*. +https://www.postgresql.org/docs/current/pgcrypto.html + +Sakana AI. (2026). *Fugu technical report*. +https://github.com/SakanaAI/fugu/blob/main/Fugu_technical_report.pdf + +*TRINITY: An evolved LLM coordinator*. (2025). arXiv. +https://arxiv.org/abs/2512.04695 diff --git a/docs/doctoring/pr-exact-head-workflows.md b/docs/doctoring/pr-exact-head-workflows.md new file mode 100644 index 000000000..39f82e99f --- /dev/null +++ b/docs/doctoring/pr-exact-head-workflows.md @@ -0,0 +1,73 @@ +# Pull-request exact-head workflow evidence + +## Decision + +Contextual Orchestrator's repository-local Tests, Fuzz, and Security workflows must run for pull requests targeting any repository branch, including stacked feature branches, and every checkout in those workflows must select the pull request's exact contributor-head SHA. + +The local exact-head checks answer one narrow question: whether the immutable contributor head passes the repository's tests, fuzzing, and security controls. They do not prove that the head integrates cleanly with its target base. Merge-tree compatibility, trusted central coverage, independent review, and branch protection remain separate mandatory evidence surfaces. + +## Problem + +The inherited workflows filtered `pull_request` events to base branch `main`. Stacked pull requests whose base is another protected integration branch therefore received no local Tests, Fuzz, or Security run. In addition, an `actions/checkout` step without an explicit pull-request head ref uses the event's default ref; GitHub documents `GITHUB_SHA` for `pull_request` as the last merge commit of `refs/pull//merge`, not the contributor-head commit. + +That combination created two evidence gaps: + +1. stacked pull requests could have no repository-local validation at all; and +2. runs that did occur could validate GitHub's generated merge commit rather than the exact branch head named by reviews, commit statuses, and dependency receipts. + +Synthetic-merge evidence can be useful for integration testing, but it cannot substitute for exact-head evidence when approvals, supply-chain receipts, and cross-repository dependencies are bound to a literal commit SHA. + +## Implemented contract + +The three local workflows now: + +- retain `push` execution only for protected `main`; +- listen to `pull_request` without a base-branch filter so stacked pull requests receive checks; +- set every checkout ref to `${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}`; +- keep `persist-credentials: false` on every checkout; +- preserve read-only top-level token permissions; +- preserve the existing same-repository guard for the CodeQL job that needs `security-events: write`; and +- do not add secrets, model credentials, repository writes, branch writers, or `pull_request_target` execution. + +`tests/test_pr_exact_head_workflows.py` fails when any of these workflows restores a `main`-only pull-request filter, omits the exact-head ref from a checkout, or persists checkout credentials. + +## Trust boundary + +The workflow definition continues to use the `pull_request` event, whose fork and Dependabot executions receive GitHub's restricted token and secret treatment. This change does not introduce `pull_request_target`, elevate a pull-request job to a privileged mutation plane, or authorize untrusted branch code to publish approvals or releases. + +Executing pull-request-controlled tests always processes untrusted code. Accordingly, repository-local workflows remain isolated from model credentials and release credentials. Their outputs are test and scanner evidence only. A passing local run does not authorize merge, approval, release, central workflow mutation, or downstream acceptance. + +## Exact-head and merge-tree evidence are complementary + +Local exact-head workflows intentionally test the contributor head itself. A later trusted integration gate must separately materialize or otherwise verify the head against its current base. Neither surface may replace the other: + +- exact-head success does not prove base compatibility; +- synthetic-merge success does not prove the literal reviewed head independently passed; +- a cancelled, queued, absent, stale-head, or predecessor-head run proves neither; and +- branch movement invalidates both review and check evidence until the new exact head reruns. + +## Test-first lineage + +Commit `4de7eee05430a8d7d9b0172ed9a59ee17b3db34d` introduced the permanent regression before the workflow repairs. The inherited workflows still contained `pull_request.branches: [main]` and omitted the required exact-head checkout ref, so the contract was RED by inspection. + +Commits `7fa66b449c47423f5c7048afe538c4afad29c4a6` and `e10a1beb87a9ca6b48a2ad8878db513278c10b36` removed the base filter and bound every Tests, Fuzz, and Security checkout to the exact contributor head. A separately materialized networkless run of the focused contract reported three passing cases. Repository GitHub Checks remain authoritative for the complete exact head. + +## Failure handling + +When an exact-head local workflow fails, inspect that exact run and repair the product or permanent workflow contract test-first. Do not reinterpret a synthetic-merge predecessor run as success. + +When a workflow is cancelled after branch movement, treat it as superseded evidence and inspect the new head's run. When the current exact-head run is cancelled without branch movement, rerun the current job only after determining that the cancellation was infrastructure- or operator-caused rather than a hidden product failure. + +When a stacked branch cannot run because of repository policy, keep the pull request Draft and correct the policy or workflow through its repository-owned maintenance path. Do not widen credentials or move the job to `pull_request_target` merely to obtain a green check. + +## Rollback + +Rollback requires reverting the workflow changes and this contract together. A partial rollback that restores a `main`-only pull-request filter or implicit merge-ref checkout recreates an evidence gap and must fail review. Preserve the read-only token boundary, immutable action pins, and non-persisted checkout credentials in every rollback. + +## APA 7 references + +GitHub. (n.d.). *Events that trigger workflows*. Retrieved August 7, 2026, from https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows + +GitHub. (n.d.). *Securely using pull_request_target*. Retrieved August 7, 2026, from https://docs.github.com/en/actions/reference/security/securely-using-pull_request_target + +GitHub. (n.d.). *Triggering a workflow*. Retrieved August 7, 2026, from https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow diff --git a/docs/doctoring/provider-credential-revocation-boundary.md b/docs/doctoring/provider-credential-revocation-boundary.md new file mode 100644 index 000000000..190827509 --- /dev/null +++ b/docs/doctoring/provider-credential-revocation-boundary.md @@ -0,0 +1,74 @@ +# Provider Credential Revocation Boundary + +## Status and scope + +This doctoring note defines the fail-closed credential boundary for outbound HTTPS requests made by Contextual Orchestrator to OpenAI-compatible model providers. It applies to the DNS-pinned provider transport used by `ModelClient` and is intentionally narrower than provider-side credential lifecycle management. + +The runtime resolves provider secrets from the configured KV backend. Environment variables are not a runtime provider-secret source. The transport uses the HTTP `Authorization` field with the `Bearer` authentication scheme for provider requests. + +## Threat model + +Provider validation and provider dispatch are separate operations. A credential can be valid when `_validate_provider()` approves the provider endpoint and then be revoked, deleted, or replaced before a later request is serialized. + +Before this boundary was added, each dispatch path resolved the credential again but converted a missing value to an empty string while constructing the request. That preserved secret non-disclosure, but it still allowed the HTTPS connection path to proceed with an empty `Authorization: Bearer ` field. The result was an unauthorized outbound network operation after revocation rather than a fail-closed configuration error. + +The security property is therefore stronger than “do not leak the old secret”: + +> If the provider credential is unavailable at dispatch time, no provider socket may be opened for that request. + +This property applies to chat completion, streaming chat, Responses API calls, batch upload, batch metadata operations, and batch content download because all external HTTPS provider egress converges on the DNS-pinned connection class. + +## Enforced boundary + +`_PinnedHTTPSConnection.request()` is the last application-controlled operation before Python's standard HTTPS request machinery can open a socket. It now requires a present, non-empty Bearer credential before delegating to `http.client.HTTPSConnection.request()`. + +The guard is deliberately located at this final pre-socket boundary as defense in depth. `ModelClient` still resolves the current KV value when it builds each request, but the connection layer refuses to turn a missing or empty current value into unauthenticated network traffic. + +The failure is `NotConfigured` and the error message does not contain the credential value. The connection object is closed before the exception is raised. No DNS pin, provider hostname, request body, or authorization value is changed by this guard when a non-empty Bearer credential is present. + +### Authorization semantics + +RFC 9110 defines HTTP field names as case-insensitive and defines the `Authorization` request field within the HTTP authentication framework. The guard therefore locates the authorization field case-insensitively. RFC 6750 defines the Bearer request-header form as the `Bearer` scheme followed by a non-empty credential. Contextual Orchestrator uses that wire form for its OpenAI-compatible provider secret even when the underlying secret is an API key rather than an OAuth access token; RFC 6750 is cited for the Bearer transport grammar, not to claim that every provider key is an OAuth token. + +RFC 9700 is the current IETF Best Current Practice updating OAuth 2.0 security guidance. Its emphasis on protecting bearer credentials, limiting token misuse, and maintaining end-to-end TLS is directionally consistent with this fail-closed boundary. The implementation does not claim OAuth conformance beyond the HTTP Bearer transport form used by the provider interface. + +## Authority and compatibility boundary + +The normal external-provider contract remains HTTPS-only, DNS-pinned, proxy-bypassing, redirect-rejecting, and certificate-verified against the original provider hostname. Credential revocation handling does not weaken any of those controls. + +The private plain-HTTP loopback seam remains a narrowly scoped local integration/test capability. Production provider validation rejects plain HTTP before external credentials are dispatched, so the pre-socket Bearer guard is intentionally implemented on the external DNS-pinned HTTPS connection rather than changing local loopback behavior. + +If a future provider transport requires an authentication scheme other than Bearer, it must introduce that scheme explicitly with provider-specific tests and an equivalent fail-closed pre-socket credential check. Removing this guard merely to make a non-Bearer provider work is not an acceptable compatibility fix. + +## Regression evidence + +`tests/test_provider_credential_revocation.py` provides three bounded contracts: + +1. A provider credential is valid during DNS validation, the active KV backend is then replaced with one that does not contain the credential, and all six external provider request paths must raise `NotConfigured` before `socket.create_connection` can execute. +2. A direct DNS-pinned request with no authorization evidence must fail before the base HTTPS request method is called. +3. A request carrying a non-empty Bearer credential must delegate unchanged to the standard HTTPS request machinery. + +The test-first lineage is preserved in pull-request history: the regression was introduced before the production transport guard. Repository-local workflow results prove only the exact commit they ran against and do not substitute for central coverage, security review, branch protection, or independent approval. + +## Operator behavior + +A `NotConfigured` failure at this boundary means the request was intentionally prevented from reaching the provider because current authentication evidence was unavailable. Operators should: + +1. verify that the configured KV backend is the intended backend for the runtime; +2. verify that the agent's credential name exists and contains a non-empty current value; +3. complete the intended rotation or restore the credential through the approved secret-management path; and +4. retry only after the credential state is correct. + +Do not work around this failure by adding an environment fallback, a blank credential, a proxy exception, disabling TLS verification, or changing the provider URL to plain HTTP. + +## Rollback + +Rollback is appropriate only if the provider authentication contract itself is intentionally redesigned. A rollback must preserve the core invariant that unavailable current credential evidence cannot open an external provider socket. Any replacement must include equivalent regression tests for every provider egress family before this guard is removed. + +## References + +Fielding, R., Nottingham, M., & Reschke, J. (Eds.). (2022). *HTTP semantics* (RFC 9110; STD 97). Internet Engineering Task Force. https://doi.org/10.17487/RFC9110 + +Jones, M., & Hardt, D. (2012). *The OAuth 2.0 authorization framework: Bearer token usage* (RFC 6750). Internet Engineering Task Force. https://doi.org/10.17487/RFC6750 + +Lodderstedt, T., Bradley, J., Labunets, A., & Fett, D. (2025). *Best current practice for OAuth 2.0 security* (RFC 9700; BCP 240). Internet Engineering Task Force. https://doi.org/10.17487/RFC9700 diff --git a/docs/doctoring/provider-json-trust-boundary.md b/docs/doctoring/provider-json-trust-boundary.md new file mode 100644 index 000000000..6fa26f39d --- /dev/null +++ b/docs/doctoring/provider-json-trust-boundary.md @@ -0,0 +1,95 @@ +# Provider JSON trust boundary + +## Decision + +Treat every complete document returned by a validated HTTPS model-provider request as untrusted structured data until the transport layer has bounded, decoded, and validated it. Normal provider endpoints must cross the boundary as one strict UTF-8 JSON object. OpenAI-compatible Batch API file-content responses remain JSON Lines, with every non-empty line independently required to be a strict JSON object. + +The transport rejects malformed UTF-8, malformed JSON, non-finite numeric extensions (`NaN`, `Infinity`, and `-Infinity`), syntactically valid floating-point exponents that overflow Python's runtime representation to a non-finite value, duplicate object member names, and top-level non-object values. It replaces decoder failures with stable redacted `RuntimeError` messages without exception chaining. Valid objects are re-serialized before existing model-client parsing so later decoder exceptions cannot retain the original untrusted document. + +This boundary supplements, rather than replaces, the existing cumulative 8 MiB provider-response byte budget and HTTP framing checks. + +## Why this is a security and diligence boundary + +A model-provider response can contain customer prompts, model output, tool arguments, retrieved business data, or upstream diagnostics. Python's `JSONDecodeError` exposes the parsed document through its `doc` attribute. Chaining that exception through retry or logging code can therefore turn malformed upstream content into durable diagnostic evidence containing private data. + +Python also deliberately accepts several behaviors that are broader than interoperable JSON: its decoder accepts `NaN`, `Infinity`, and `-Infinity`, and repeated object names default to last-value-wins semantics. In addition, the default binary-float conversion can materialize an extreme but syntactically valid exponent such as `1e999` as infinity. Those behaviors are useful for general-purpose compatibility but are undesirable at an orchestration trust boundary because two implementations can interpret the same provider evidence differently. + +RFC 8259 requires JSON exchanged between systems outside a closed ecosystem to use UTF-8, excludes literal non-finite values from the JSON number grammar, states that object names should be unique for interoperable behavior, and specifically identifies values such as `1E400` as evidence of potential interoperability problems when they exceed commonly available binary64 range and precision. The Python 3.14 documentation explicitly documents the non-finite-number extension and default repeated-name behavior, and exposes `parse_constant`, `parse_float`, and `object_pairs_hook` as controls. The implementation uses those controls to fail closed before a finite-syntax number can become a non-finite orchestration value. + +## Request-path authority + +The DNS-pinned HTTPS connection retains the exact request target after the final Bearer-credential check and before dispatch. The paired response wrapper uses that already-reviewed target only to distinguish response representation: + +- `/files/{file_id}/content` paths are treated as Batch-compatible JSON Lines; +- every other complete validated-provider response consumed through the model client is treated as one JSON object; and +- streaming responses continue through the separate `text/event-stream` iterator boundary, where each `data:` event is now validated as a strict JSON object until `data: [DONE]`. + +The request path is not a new routing authority and does not change the destination, DNS pin, TLS hostname, credentials, or provider selection. It is metadata carried from the validated outbound request to the response parser on the same connection. + +## Batch compatibility + +The OpenAI Batch API defines batch input and output as per-line request/output objects stored in JSONL files. Batch file-content responses therefore cannot be forced through a single-document JSON parser. The transport validates every non-empty row independently, requires an object per row, rejects duplicate names and any numeric value that would become non-finite at the Python boundary, and emits canonical UTF-8 JSON Lines for the existing batch parser. + +Blank-only or malformed output fails closed. A malformed provider row is never handed to the later orchestration-level `json.loads` call, so provider-controlled text is not retained in that later exception's document field. + +## Failure semantics + +The boundary uses intentionally small, stable messages: + +- `provider JSON response is malformed` for invalid UTF-8, invalid JSON syntax, duplicate names, non-finite extensions, float overflow to a non-finite runtime value, or parser/encoder recursion failure; +- `provider JSON response must be an object` when a valid JSON document has the wrong top-level type; and +- `provider JSON Lines response is malformed` for invalid Batch output content. + +The messages do not contain provider text, parsed values, URLs with credentials, decoder offsets, or the underlying exception. Public provider retry code may wrap these stable exceptions, but the wrapped cause is already redacted and contains no original provider document. + +## Resource and privacy properties + +The response wrapper continues to read at most the configured cumulative response budget before parsing. Strict parsing therefore cannot turn an unbounded response into an unbounded allocation path. Canonicalization can temporarily hold the bounded parsed representation and encoded representation in memory; the byte budget remains the admission control for provider-controlled body size. + +No raw provider body is added to logs, audit records, workflow traces, exception messages, or review evidence by this change. The implementation does not introduce telemetry fields, persistent state, database objects, or new credentials. + +## Provider and environment boundaries + +Validated production provider traffic uses HTTPS, DNS pinning, TLS hostname verification, proxy bypass, redirect rejection, and the captured request target. The plain-HTTP literal-loopback path remains an explicit integration/development seam and is not promoted to production provider authority. Lightweight response doubles without a captured validated request target retain historical byte-oriented `read()` behavior so framing and resource tests can remain isolated; focused callers can exercise `read_json_object()` directly. + +Live NVIDIA NIM development continues to use the repository credential abstraction with `NVIDIA_NIM_API_KEY`. This change neither reads `COPILOT_GITHUB_TOKEN` nor changes independent reviewer credentials or identity. + +## Verification + +`tests/test_provider_json_boundary.py` and `tests/test_provider_json_finite_number_boundary.py` prove that: + +1. malformed UTF-8 and malformed JSON are redacted and carry no original exception cause; +2. `NaN`, positive/negative infinity, duplicate names, top-level arrays, and extreme exponents that overflow to infinity are rejected; +3. valid Unicode JSON objects and ordinary finite exponent notation survive the boundary; +4. the pinned HTTPS connection records the exact provider request target; +5. chat, Responses passthrough, file-upload metadata, and batch metadata paths fail closed before their existing `json.loads` calls can retain malformed provider documents; +6. valid structured responses preserve existing caller semantics; +7. Batch output remains line-addressable JSONL after strict validation; +8. malformed Batch JSONL is redacted and resources close deterministically; and +9. explicit partial reads remain bounded byte operations and are not parsed prematurely. + +Repository-local exact-head GitHub Checks remain diagnostic for the contributor head only. They do not replace trusted central 100% production statement/branch/public-docstring/package evidence, fresh required review-agent verdicts, qualifying independent approval, branch protection, or protected-main acceptance. + +## Operator handling + +When this boundary rejects a provider response: + +1. identify the provider and endpoint from existing request/audit metadata rather than logging the response body; +2. confirm whether the upstream service returned malformed JSON, duplicate names, a non-finite extension or out-of-range exponent, an unexpected top-level type, or malformed Batch JSONL; +3. validate the provider against its current documented OpenAI-compatible contract; +4. reproduce with synthetic non-sensitive data when evidence is needed; and +5. fix the provider or compatibility adapter rather than relaxing the global parser. + +A provider-specific compatibility exception requires a reviewed architectural decision and dedicated regression tests. Do not re-enable permissive Python JSON extensions globally or admit numeric values that become non-finite in the runtime representation. + +## Rollback + +If a conforming provider is incorrectly rejected, revert the parser change only after preserving the existing response-size, DNS-pinning, redirect, proxy, credential, TLS, and redaction protections. A rollback must not restore raw `JSONDecodeError`/`UnicodeDecodeError` propagation across the provider trust boundary. Prefer a narrowly documented provider adapter that converts a verified non-standard representation before orchestration logic consumes it. + +## References + +Bray, T. (2017). *The JavaScript Object Notation (JSON) Data Interchange Format* (RFC 8259). Internet Engineering Task Force. https://doi.org/10.17487/RFC8259 + +OpenAI. (n.d.). *Batch API reference*. Retrieved August 8, 2026, from https://platform.openai.com/docs/api-reference/batch + +Python Software Foundation. (2026). *json — JSON encoder and decoder (Python 3.14.6 documentation).* https://docs.python.org/3/library/json.html diff --git a/docs/doctoring/provider-response-resource-bound.md b/docs/doctoring/provider-response-resource-bound.md new file mode 100644 index 000000000..09464ea55 --- /dev/null +++ b/docs/doctoring/provider-response-resource-bound.md @@ -0,0 +1,109 @@ +# Provider response resource-bound evidence + +## Decision record + +Contextual Orchestrator treats every model-provider response body and its framing metadata as untrusted input. The DNS-pinned transport therefore limits cumulative consumed response bytes to **8 MiB (8,388,608 bytes) per HTTP response** before JSON parsing, batch-output decoding, or server-sent-event processing can continue. + +The 8 MiB value is a reviewed product safety bound, not a value mandated by HTTP, OWASP, or Python. It is intentionally high relative to ordinary chat/completion responses while remaining low enough to prevent one provider response from becoming an unbounded memory-consumption surface. Workloads whose legitimate batch output exceeds the bound must be partitioned into smaller batches rather than silently truncating evidence. + +## Security and reliability basis + +OWASP API4:2023 identifies missing or inappropriate resource limits as a common API weakness with denial-of-service and cost consequences, and recommends explicit bounds on data size and resource consumption. This applies to outbound provider integration as well as inbound APIs because a compromised, malfunctioning, or policy-incompatible upstream can return unexpectedly large data. + +RFC 9110 defines the semantics of HTTP response content but does not impose an application-specific maximum representation size. It permits a recipient to normalize repeated `Content-Length` values only when every decimal value is identical, requires invalid or conflicting values to be rejected, and warns that recipients must anticipate potentially large decimal numerals. The application therefore owns both a framing-validation policy and a defensible consumption limit. + +Python's `http.client.HTTPResponse.read([amt])` supports reading at most the next requested number of bytes. Contextual Orchestrator uses that primitive to request no more than the remaining response budget plus one byte. The single probe byte distinguishes an exact-limit response from an oversized response without first buffering the complete body. Python's `HTTPResponse.getheader()` combines repeated values with a comma, so the transport parses the complete field value rather than trusting only its first member. + +The WHATWG HTML Standard defines the server-sent-event stream format's MIME type as `text/event-stream`, requires UTF-8 encoding, and specifies event-stream line framing. A successful HTTP status alone therefore does not prove that a provider honored the streaming representation contract. A reverse proxy, provider fallback, or incompatible endpoint can return a `200` response with JSON, HTML, or another representation. Contextual Orchestrator requires the normalized response media type to be `text/event-stream` before it consumes a real provider response through the streaming iterator; media-type parameters such as `charset=utf-8` remain compatible. + +OpenAI's Chat Completions API additionally defines an OpenAI-compatible streaming terminal contract: streamed chunks are followed by a `data: [DONE]` message, while interruption can prevent final stream metadata from arriving. Contextual Orchestrator therefore treats `[DONE]` as positive completion evidence for an accepted OpenAI-compatible SSE response instead of treating transport EOF as successful model completion. + +## Runtime contract + +`contextual_orchestrator.provider_transport._ProviderHTTPResponse` owns the bound for every existing `ModelClient` provider path. No caller-specific opt-in is required. + +The contract is: + +1. a declared `Content-Length` greater than the remaining 8 MiB response budget is rejected before any body byte is read; +2. repeated comma-separated `Content-Length` members are accepted only when their ASCII decimal values are canonically equal, including harmless leading-zero differences, with only HTTP optional whitespace (`SP` or `HTAB`) removed around each member; +3. empty, signed, fractional, non-ASCII, non-HTTP-whitespace, malformed, or conflicting `Content-Length` evidence fails closed; +4. simultaneous `Content-Length` and `Transfer-Encoding` evidence is rejected as ambiguous rather than selecting a preferred framing interpretation; +5. decimal lengths are compared as canonical strings instead of converting an attacker-controlled numeral into an unbounded integer; +6. absent `Content-Length` remains supported and is still governed by cumulative consumption accounting; +7. ordinary `read()` calls request at most the remaining budget plus one byte; +8. explicit smaller reads preserve the caller's requested size; +9. explicit negative or over-budget reads are reduced to the remaining budget plus one byte; +10. cumulative reads share one budget for the lifetime of the response wrapper; +11. real `http.client.HTTPResponse` iteration is treated as provider streaming and requires a normalized `Content-Type` media type of exactly `text/event-stream` before any body line is consumed; +12. `text/event-stream` parameters are accepted after media-type normalization, while missing or different media types fail closed instead of producing an empty or misframed successful stream; +13. real provider streaming uses size-limited `readline()` calls so a single pathological SSE line cannot bypass the bound before inspection; +14. an accepted SSE response is successful only when its OpenAI-compatible terminal `data: [DONE]` marker is consumed; +15. JSON-malformed `data:` frames fail closed instead of being silently discarded; +16. EOF before `[DONE]` fails closed instead of converting a truncated model answer into successful orchestration evidence; +17. lightweight non-HTTP test doubles retain ordinary iteration while receiving the same cumulative accounting; +18. exceeding 8 MiB raises a non-transient runtime failure rather than returning truncated content; and +19. constructor-time framing rejection and context-managed consumption failure both close the response and direct provider connection deterministically. + +The header preflight is an optimization and an early rejection boundary, not proof that the eventual body is trustworthy or complete. A provider can omit or misstate its declared length, so every accepted response remains subject to the authoritative cumulative byte counter. + +This boundary applies uniformly to chat responses, raw OpenAI-compatible passthrough, batch metadata, batch result downloads, and SSE streaming because those paths all consume the same response wrapper. Ordinary JSON and batch-response bodies use bounded `read()` and retain their existing completion semantics. Real HTTP response iteration is reserved for the provider-streaming path and therefore requires `text/event-stream` plus `[DONE]` completion evidence. + +## Failure semantics + +Oversize provider content, invalid or ambiguous framing, and an incompatible provider-stream media type are not retried as transient network errors. Retrying the same policy-invalid representation would amplify provider load, duplicate spend, and resource consumption without changing the violated contract. + +Framing-header and stream-media-type header access failures become stable redacted runtime errors. Provider-controlled exception text is not exposed to callers or logs through this boundary. Cleanup is attempted by the response context manager, and cleanup failure cannot convert the invalid provider response into successful orchestration evidence. + +The service does not publish partial JSON, partial batch evidence, or a partial streamed line after the byte budget is exceeded. A wrong or missing streaming media type is rejected before the first body line is consumed. For a valid live SSE connection, deltas delivered before a later malformed frame or premature EOF cannot be recalled from an already-reading client. The provider stream nevertheless fails rather than being persisted as a successful route, and the HTTP streaming surface can terminate with its existing error semantics instead of presenting the truncated provider response as complete. + +A malformed provider `data:` payload is reported through a stable redacted runtime error; provider-controlled parser detail is not exposed through this boundary. Premature EOF is likewise reported as an incomplete provider stream rather than accepted as implicit completion. + +## Verification contract + +`tests/test_provider_response_bounds.py` preserves the resource-bound regression surface. `tests/test_true_streaming.py` preserves the provider-stream completion and media-type regression surface. Together they prove: + +- the reviewed default is exactly 8 MiB; +- invalid byte budgets fail closed; +- over-budget declared lengths fail before a body read; +- equal repeated decimal lengths, including leading-zero equivalents and valid `SP`/`HTAB` optional whitespace, remain valid; +- malformed, non-ASCII, vertical/form-feed whitespace, non-breaking-space, and conflicting declared lengths fail closed; +- `Content-Length` plus `Transfer-Encoding` fails as ambiguous; +- framing-header lookup failures are redacted and close both resources; +- an unbounded read probes only one byte beyond the remaining budget; +- bodies exactly at the limit remain valid; +- repeated explicit reads cannot bypass cumulative accounting; +- negative full-read semantics remain bounded; +- valid `text/event-stream` responses, including media-type parameters, use bounded `readline()` calls; +- a `200` provider response with a non-event-stream media type is rejected instead of masquerading as a successful stream; +- stream `Content-Type` lookup failure is redacted and closes response and connection resources; +- cumulative streaming overflow fails; +- a normal provider SSE sequence preserves live content deltas and accepts `[DONE]` completion; +- malformed provider `data:` JSON fails closed; +- EOF before `[DONE]` fails closed after any previously delivered deltas; and +- response and connection cleanup still occurs when framing or consumption validation fails. + +The streaming media-type regression is test-first: commit `f4f650734aa0bad856b3e4389d63de45d0e52e76` adds the wrong-media-type failure contract before production commit `529182d9186fd353cb1cd44e08f9483c39520cad` enforces it. The earlier SSE-completion and response-bound test-only commits likewise precede their production implementations. Focused deterministic tests require no provider credential or public network egress. Repository and central exact-head gates remain separately authoritative. + +## Operational and compatibility notes + +The bound and media-type validation are intentionally enforced below model- or provider-specific parsing so malformed content cannot claim a larger allowance or a successful streaming interpretation by choosing another response shape. No provider identity, model-routing policy, reviewer credential, or authority boundary changes. + +A provider that sends both `Content-Length` and `Transfer-Encoding`, conflicting repeated lengths, non-decimal length evidence, whitespace outside HTTP `SP`/`HTAB`, a missing or non-`text/event-stream` streaming media type, malformed OpenAI-compatible SSE `data:` JSON, or an SSE connection that ends without `[DONE]` is treated as incompatible or incomplete. Operators should preserve the bounded failure, retain the provider and endpoint identity in redacted incident evidence, and correct or replace the upstream integration rather than adding a local parsing exception. + +If production evidence shows a legitimate response class consistently approaching 8 MiB, raise a separately reviewed change with workload measurements and an explicit threat-model update. Do not disable the bound locally, truncate silently, or add an unreviewed environment-variable bypass. Likewise, do not reinterpret a JSON/HTML `200` response or transport EOF as successful OpenAI-compatible streaming completion merely to accommodate an incompatible provider; such compatibility must be a separately reviewed protocol contract with explicit provider isolation. + +## Rollback + +If a compatibility regression is discovered, revert the framing-preflight and bounded-response commits as reviewed units and preserve the prior DNS-pinning, TLS identity, redirect, proxy-isolation, and cleanup controls. If the SSE media-type or completion hardening must be reverted separately, keep the byte budget intact and document which provider contract cannot emit the standard `text/event-stream` representation or OpenAI-compatible terminal marker. A rollback is not authorization to introduce an unbounded alternative transport, accept ambiguous HTTP framing, or silently bless a non-stream representation or partial model output as complete. + +## APA 7 references + +Fielding, R., Nottingham, M., & Reschke, J. (Eds.). (2022). *HTTP semantics* (RFC 9110; STD 97). Internet Engineering Task Force. https://www.rfc-editor.org/rfc/rfc9110.html + +OpenAI. (n.d.). *Chat Completions*. OpenAI API Reference. Retrieved August 8, 2026, from https://platform.openai.com/docs/api-reference/chat + +OWASP Foundation. (2023). *API4:2023 unrestricted resource consumption*. OWASP API Security Top 10. https://owasp.org/API-Security/editions/2023/en/0xa4-unrestricted-resource-consumption/ + +Python Software Foundation. (2026). *http.client — HTTP protocol client*. Python 3.14.6 documentation. https://docs.python.org/3.14/library/http.client.html + +WHATWG. (2026, July 20). *HTML Standard, Edition for Web Developers: Server-sent events*. https://html.spec.whatwg.org/dev/server-sent-events.html diff --git a/docs/doctoring/provider-stream-utf8-boundary.md b/docs/doctoring/provider-stream-utf8-boundary.md new file mode 100644 index 000000000..aaba5f1d2 --- /dev/null +++ b/docs/doctoring/provider-stream-utf8-boundary.md @@ -0,0 +1,44 @@ +# Provider stream UTF-8 boundary + +## Decision + +Contextual Orchestrator treats provider server-sent-event bytes as untrusted network input. A real provider stream is accepted only when its response uses the `text/event-stream` media type, every consumed line stays within the existing cumulative response-byte budget, and each line is valid UTF-8 before any event-data interpretation occurs. + +If one consumed SSE line is not valid UTF-8, orchestration fails closed with the stable public error `malformed provider stream event`. The original `UnicodeDecodeError` is deliberately suppressed so provider-controlled bytes, byte positions, or decoder diagnostics are not carried into ordinary exception text. The response and direct connection are still closed by the existing response context boundary. + +This rule does not change provider routing, model selection, retry policy, credential resolution, DNS pinning, TLS identity, proxy rejection, redirect rejection, the 8 MiB response budget, or the OpenAI-compatible `[DONE]` terminal-marker contract. + +## Why this is required + +The WHATWG HTML Standard defines the server-sent-event stream format as `text/event-stream` and states that event streams are always decoded as UTF-8. RFC 8259 independently requires JSON exchanged between systems outside a closed ecosystem to use UTF-8. Because this runtime validates JSON-bearing `data:` frames from external model providers, malformed UTF-8 is protocol-invalid input rather than recoverable text. + +Failing before JSON parsing also keeps the error boundary deterministic. Python's native `UnicodeDecodeError` may retain the offending byte sequence and byte offsets. Those diagnostics are useful inside a controlled parser test but are unnecessarily detailed at the provider trust boundary, where response content may be confidential, adversarial, or both. + +## Verification + +The regression was introduced test-first at commit `76fc98ba720632950df732cb38c1f58df4e42b0b`. The test supplies an `HTTPResponse`-shaped `text/event-stream` response containing malformed UTF-8 plus recognizable provider-controlled text and requires: + +- a stable `RuntimeError` classification; +- no provider-controlled text in the public exception string; +- deterministic response cleanup; and +- deterministic connection cleanup. + +The production repair at commit `e46a9d894a3951a991d177c879eb0b9247882cea` catches only `UnicodeDecodeError` at the UTF-8 decoding boundary and re-raises the existing malformed-stream error without exception chaining. All byte-budget, framing, media-type, JSON, and terminal-marker checks remain independently enforced. + +Repository GitHub Checks on the final exact head remain authoritative. A predecessor-head or local diagnostic result does not authorize merge. + +## Failure and rollback boundary + +If a provider emits malformed UTF-8, operators should treat the response as a provider/protocol failure and inspect provider-side telemetry under the provider's own confidentiality controls. The orchestrator must not relax decoding to replacement characters or another character encoding because doing so would admit data outside the standardized SSE/JSON interoperability contract. + +Rollback consists of reverting the production and regression commits together. Removing only the regression would weaken assurance; removing only the production guard would intentionally restore the demonstrated disclosure-prone exception path. + +## Authority boundary + +This control validates transport syntax only. It does not establish truthfulness, safety, authorization, model identity, provenance, semantic correctness, or successful completion of model output. Hosts and downstream services retain their existing authorization, tenancy, retention, audit, and model-use responsibilities. + +## References + +Bray, T. (Ed.). (2017). *The JavaScript Object Notation (JSON) Data Interchange Format* (RFC 8259). Internet Engineering Task Force. https://doi.org/10.17487/RFC8259 + +WHATWG. (2026, July 16). *HTML Standard: Server-sent events*. https://html.spec.whatwg.org/dev/server-sent-events.html diff --git a/docs/doctoring/provider-transfer-encoding-boundary.md b/docs/doctoring/provider-transfer-encoding-boundary.md new file mode 100644 index 000000000..ff0d69b11 --- /dev/null +++ b/docs/doctoring/provider-transfer-encoding-boundary.md @@ -0,0 +1,56 @@ +# Provider Transfer-Encoding boundary + +## Decision + +Contextual Orchestrator accepts only a single HTTP/1.1 `chunked` `Transfer-Encoding` on model-provider responses. Any other transfer-coding value or chain fails closed before application JSON, batch, or server-sent-event parsing begins. `Content-Length` together with any `Transfer-Encoding` remains an ambiguous-framing error under the existing response-bound contract. + +This is intentionally stricter than the full HTTP/1.1 protocol. RFC 9112 permits response transfer-coding chains such as `gzip, chunked` when `chunked` is final, and also defines close-delimited behavior when a non-chunked coding is final. The repository does not implement a reviewed transfer-decoding stack for arbitrary codings. Passing such bytes through to higher-level provider parsers would therefore make application semantics depend on transport metadata that this client did not decode or validate. + +Python's `http.client` is the repository's reviewed transport primitive and directly supports HTTP/1.1 chunked framing. The product consequently keeps the interoperable single `chunked` case and rejects unsupported chains instead of silently treating transfer-coded bytes as ordinary provider content. + +## Security and reliability rationale + +RFC 9112 makes message framing security-sensitive, identifies `Transfer-Encoding`/`Content-Length` disagreement as a potential smuggling or response-splitting signal, forbids applying `chunked` more than once, and requires recipients to understand chunked framing. A provider response can be syntactically valid HTTP while still being outside this application's decoder contract. The safe application boundary is therefore: support only the coding the selected standard-library transport decodes and fail closed on every other coding before model-output interpretation. + +The cumulative 8 MiB response budget remains authoritative after chunk decoding. This change does not raise the byte limit, accept ambiguous framing, add a proxy, enable redirects, alter DNS pinning, weaken TLS hostname verification, or introduce a provider-specific exception. + +## Runtime contract + +`contextual_orchestrator.provider_transport._ProviderHTTPResponse` applies the following order to real `http.client.HTTPResponse` objects: + +1. read `Content-Length` and `Transfer-Encoding` through the existing redacted header-inspection boundary; +2. reject the response when both fields are present; +3. when `Transfer-Encoding` is present without `Content-Length`, accept only a case-insensitive field value exactly equal to `chunked`; +4. reject empty values, alternative codings, coding chains, repeated `chunked`, or parameterized `chunked` as unsupported by the product decoder; +5. retain the existing `Content-Length` validation when no transfer coding is present; and +6. retain cumulative bounded reads, bounded SSE lines, media-type validation, terminal `[DONE]` evidence, and deterministic cleanup after framing admission. + +The policy distinguishes protocol validity from product support. `gzip, chunked` can be valid HTTP/1.1 and is still rejected here because Contextual Orchestrator has no reviewed transfer-decoding layer for the preceding `gzip` coding. Compatibility must be added through a separately reviewed decoder with equivalent resource, integrity, and test coverage rather than by weakening this gate. + +## Verification + +Test-first commit `d3d02f33d26511707b6686edc38e5211aa490828` introduced `tests/test_provider_transfer_encoding.py` before the production gate existed. Production commit `7dc4ae62ba0f4c0e6dd2b707dae163ee01f648a2` implements the fail-closed boundary. + +The regression contract covers: + +- accepted `chunked` casing variants; +- rejected `gzip`; +- rejected valid-but-unsupported `gzip, chunked`; +- rejected repeated `chunked, chunked`; +- rejected parameterized `chunked;foo=bar`; +- rejected `identity` and an empty field value; and +- deterministic response and connection cleanup for every rejected coding. + +Repository exact-head CI, security, coverage, review, and protected-merge gates remain authoritative. These focused tests do not substitute for the pending central coverage evidence required by PR #96. + +## Operations and rollback + +An upstream that emits a non-`chunked` transfer coding should be treated as incompatible with the current provider transport. Preserve the provider identity and fixed failure classification in incident evidence, then correct the upstream or introduce an explicitly reviewed decoding adapter. Do not add a one-off bypass, pass encoded bytes to model parsers, or reinterpret transport EOF as proof of a complete provider response. + +If a compatibility regression is confirmed, revert this bounded transfer-coding gate as a reviewed unit while preserving the existing 8 MiB byte bound, `Content-Length` validation, ambiguous-framing rejection, DNS pinning, TLS identity, redirect rejection, proxy isolation, and response cleanup controls. + +## APA 7 references + +Fielding, R., Nottingham, M., & Reschke, J. (Eds.). (2022). *HTTP/1.1* (RFC 9112; STD 99). Internet Engineering Task Force. https://www.rfc-editor.org/rfc/rfc9112.html + +Python Software Foundation. (2026). *http.client — HTTP protocol client*. Python 3.14.6 documentation. https://docs.python.org/3.14/library/http.client.html diff --git a/docs/kv-credentials.md b/docs/kv-credentials.md index 6860aeeec..d53ccc5a1 100644 --- a/docs/kv-credentials.md +++ b/docs/kv-credentials.md @@ -57,6 +57,24 @@ at bootstrap by `CONTEXTUAL_ORCHESTRATOR_KV_BACKEND`: Tests and the app suite run on the in-memory backend, so **no KV or Postgres is required to run `pytest`**. +### Durable-backend failure semantics + +Selecting a durable Postgres backend makes it authoritative. Import, +connection, initialization, and bootstrap-seed failures raise the stable, +redacted error `Postgres config backend is unavailable`; the orchestrator +never silently falls back from Postgres to process-local memory. The public +error and rendered traceback omit the DSN and its credentials. + +`CostRoutingCoordinator(postgres_dsn=...)` enters this same factory boundary +when no config store is injected, so routing policy, prices, and credential +authority cannot diverge from its Postgres-backed token-counting authority. + +Operators must restore the configured backend and restart. If losing durable +configuration, prices, routing policy, and credential authority is genuinely +acceptable for a local or test deployment, intentionally select `memory` +instead. A failed Postgres backend never triggers provider-secret lookup from +environment variables. + ### Postgres pgcrypto registry (org reference pattern) The default production backend mirrors xtrmLLMBatchPython's pgcrypto-encrypted diff --git a/docs/provider_catalog.md b/docs/provider_catalog.md new file mode 100644 index 000000000..6ec8a38c2 --- /dev/null +++ b/docs/provider_catalog.md @@ -0,0 +1,129 @@ +# Provider catalog operator guide + +Use this guide to turn the five existing organization provider secrets into the +runtime model pool without placing API-key values in source, agent JSON, or the +long-running process environment. + +## Required Actions secrets + +Provider credentials: + +- `NVIDIA_NIM_API_KEY` +- `NVIDIA_NIM_API_KEY_SUB` +- `BYTEZ_API_KEY` +- `OPENROUTER_API_KEY` +- `OPENAI_API_KEY` + +Durable registry/catalog bootstrap: + +- `CONTEXTUAL_ORCHESTRATOR_KV_DSN` +- `CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE` + +The provider keys already named above are not sufficient by themselves to create +a durable database result. The DSN and passphrase tell the trusted job where the +pgcrypto registry/catalog lives and how to decrypt credentials later. When either +is absent, the workflow fails instead of reporting success against temporary +memory. + +## First synchronization + +After the feature reaches protected `main`: + +1. Open **Actions → Provider Catalog Sync → Run workflow**. +2. Select protected `main` and the `production` environment. +3. Wait for **Seed credentials and refresh durable catalog** to finish. +4. Read only the safe summary: + - `candidate_agent_count` must be greater than zero; + - each intended account should be `refreshed`; + - `stale_available` is serviceable but requires provider investigation; + - `failed` means that account has no usable discovered model. +5. Do not copy a provider key into an issue when diagnosing a failure. Use the + credential name and stable error code. + +The same workflow runs every six hours. Each provider refresh is isolated, so one +outage does not erase other accounts or its own last-known-good models. + +## Start the gateway from the catalog + +```bash +export CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN="$CONTEXTUAL_ORCHESTRATOR_KV_DSN" +python -m contextual_orchestrator --serve \ + --provider-catalog-dsn "$CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN" \ + --admin-token "$CONTEXTUAL_ORCHESTRATOR_ADMIN_TOKEN" \ + --inference-token "$CONTEXTUAL_ORCHESTRATOR_INFERENCE_TOKEN" \ + --host 127.0.0.1 \ + --port 8000 +``` + +`--provider-catalog-dsn` is authoritative. It disables the seed agents file and +loads enabled database models. Startup fails when the database is unavailable or +contains no enabled candidate; it does not silently start `examples/agents.mock.json`. + +OpenAI, OpenRouter, and NVIDIA NIM models use the hardened OpenAI-compatible +transport. Bytez models use the native Bytez adapter. A Bytez request that needs +an unsupported Responses/tool passthrough fails closed rather than returning a +fabricated OpenAI object; another eligible provider should be selected for that +contract. + +## Confirm the pool + +Use the authenticated admin agent-pool endpoint or console and verify: + +- provider names include the accounts refreshed successfully; +- NVIDIA primary and secondary entries have different agent ids and credential + names; +- no agent JSON contains a provider key value; +- reasoning, coding, vision, audio, and embedding models carry only capabilities + supported or conservatively inferred from catalog metadata; +- unknown context and price fields remain absent/null rather than zero; +- disabled accounts are absent from runtime candidates but remain in catalog + history. + +## Respond to common failures + +### `provider credential inventory is incomplete` + +Add or repair the exact missing Actions secret, then rerun the protected workflow. +The required bootstrap performs no partial credential write. + +### `provider catalog requires a PostgreSQL DSN` + +Configure `CONTEXTUAL_ORCHESTRATOR_KV_DSN`. Do not replace it with a temporary +SQLite or memory path in production. + +### `catalog_authentication_failed` + +Rotate or reauthorize the named provider credential. The catalog client does not +retry 401/403 repeatedly. + +### `stale_available` + +The current refresh failed, but the last complete catalog remains enabled. Check +provider status, egress, entitlement, and rate limits. The next scheduled job +will retry within bounded limits. + +### `no usable provider model exists after catalog refresh` + +All enabled accounts lack both a current and prior candidate. Restore at least +one provider or the database before starting the gateway. Do not bypass this by +starting an unlabeled mock deployment. + +### Bytez response/passthrough failure + +Confirm the selected model supports ordinary native chat input. Route OpenAI +Responses, tool calling, or structured passthrough to a provider whose contract +supports it. Do not add response-shape guessing. + +## Rotation without downtime + +1. Replace the value under the existing Actions secret name. +2. Run Provider Catalog Sync manually. +3. Verify the affected account refreshes and a canary succeeds. +4. Revoke the old key. +5. Confirm the next runtime request resolves the updated registry value. + +The model catalog refers to the stable credential name, so no model-row or +consumer configuration change is needed during rotation. + +For incident handling, rollback, and evidence interpretation, read +[`docs/doctoring/durable-provider-catalog.md`](doctoring/durable-provider-catalog.md). diff --git a/docs/superpowers/plans/2026-08-05-atheris-interpreter-lock.md b/docs/superpowers/plans/2026-08-05-atheris-interpreter-lock.md new file mode 100644 index 000000000..ab734d208 --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-atheris-interpreter-lock.md @@ -0,0 +1,156 @@ +# Interpreter-Portable Atheris Lock Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the hash-locked Atheris fuzz dependency install deterministically across the repository's supported Python 3.11 and Python 3.13+ validation environments. + +**Architecture:** Keep one universal requirements lock and partition Atheris versions with standardized Python environment markers. Add a no-egress contract test that treats project metadata and the lock as immutable evidence and rejects gaps, overlap, or missing hashes. + +**Tech Stack:** Python 3.10+, `tomllib`, `pytest`, PyPA dependency specifiers, uv-generated hash locks. + +## Global Constraints + +- Do not modify provider transports, runtime behavior, reviewer identities, reviewer secrets, or workflow permissions. +- Keep `--require-hashes` compatibility and explicit SHA-256 artifacts. +- Add explanatory docstrings to every new helper and test. +- Document current authoritative sources in `docs/doctoring/` using APA 7 references. +- Update `CHANGELOG.md`. +- Use only descriptive multi-word snake_case names for new data or database objects; this slice creates no database object. + +--- + +### Task 1: Add the failing dependency-lock contract + +**Files:** +- Create: `tests/test_fuzz_dependency_lock.py` + +**Interfaces:** +- Consumes: `pyproject.toml`, `fuzz/requirements-atheris.txt` +- Produces: deterministic assertions for interpreter selection and hash completeness + +- [ ] **Step 1: Write a test that expects two mutually exclusive Atheris requirements** + +The test must load the `fuzz` extra with `tomllib`, parse the two exact Atheris entries, and evaluate representative Python 3.11, 3.13, and 3.14 environments. + +- [ ] **Step 2: Write a test that verifies lock markers and published hashes** + +Require the lock to contain the 3.0.0 and 3.1.0 entries with matching interpreter partitions and at least one SHA-256 hash per entry. + +- [ ] **Step 3: Run the focused test and verify RED** + +Run: `python -m pytest tests/test_fuzz_dependency_lock.py -q` + +Expected: FAIL because the Python 3.13+ project requirement and lock entry do not yet exist. + +- [ ] **Step 4: Commit the failing contract** + +```bash +git add tests/test_fuzz_dependency_lock.py +git commit -m "test(fuzz): require interpreter-portable Atheris lock" +``` + +### Task 2: Partition the fuzz extra by interpreter + +**Files:** +- Modify: `pyproject.toml` + +**Interfaces:** +- Produces: `atheris==3.0.0; python_version < '3.13'` and `atheris==3.1.0; python_version >= '3.13'` + +- [ ] **Step 1: Add the Python 3.13+ requirement** + +Keep the existing pre-3.13 marker and add the mutually exclusive 3.1.0 marker. + +- [ ] **Step 2: Run the focused test** + +Run: `python -m pytest tests/test_fuzz_dependency_lock.py -q` + +Expected: the metadata assertion passes; the lock assertion remains RED. + +- [ ] **Step 3: Commit** + +```bash +git add pyproject.toml +git commit -m "build(fuzz): partition Atheris by interpreter" +``` + +### Task 3: Regenerate the universal hash lock + +**Files:** +- Modify: `fuzz/requirements-atheris.in` +- Modify: `fuzz/requirements-atheris.txt` + +**Interfaces:** +- Consumes: the project interpreter partition +- Produces: one hash-locked requirements file valid for all supported runners + +- [ ] **Step 1: Document the interpreter split in the input file** + +Explain the repository Python 3.11 fuzz runner and newer coverage-evidence interpreter without referring to secret values or mutable runner identity. + +- [ ] **Step 2: Add the 3.1.0 lock entry and published hashes** + +Use the exact version markers and SHA-256 values recorded in the doctoring evidence. + +- [ ] **Step 3: Run the focused test and verify GREEN** + +Run: `python -m pytest tests/test_fuzz_dependency_lock.py -q` + +Expected: PASS. + +- [ ] **Step 4: Run the complete repository gates** + +Run the repository's documented Tests and Fuzz commands, followed by compilation and package-install smoke tests. + +- [ ] **Step 5: Commit** + +```bash +git add fuzz/requirements-atheris.in fuzz/requirements-atheris.txt +git commit -m "build(fuzz): lock Atheris for supported interpreters" +``` + +### Task 4: Record evidence and release notes + +**Files:** +- Create: `docs/doctoring/atheris-interpreter-lock.md` +- Create: `CHANGELOG.md` + +**Interfaces:** +- Produces: source-backed operational rationale and Unreleased change evidence + +- [ ] **Step 1: Add the doctoring record** + +Record the PyPA environment-marker specification, Atheris release artifacts, hash provenance, uncertainty boundary, and APA 7 references. + +- [ ] **Step 2: Initialize the changelog** + +Use Keep a Changelog structure and add the interpreter-portable lock under `Changed`. + +- [ ] **Step 3: Run documentation and full validation gates** + +Run formatting, tests, fuzzing, security, SAST, package build/install smoke tests, and docstring gates. + +- [ ] **Step 4: Commit** + +```bash +git add docs/doctoring/atheris-interpreter-lock.md CHANGELOG.md +git commit -m "docs(fuzz): record interpreter-portable lock evidence" +``` + +### Task 5: PR validation and integration + +**Files:** +- No new source files + +**Interfaces:** +- Produces: a mergeable prerequisite for PR #76 + +- [ ] **Step 1: Open a focused PR closing issue #95** + +- [ ] **Step 2: Review every automated and human finding** + +Apply only validated fixes, preserve the narrow scope, and rerun exact-head checks. + +- [ ] **Step 3: Merge only after all required checks and reviews pass** + +- [ ] **Step 4: Update PR #76 to the new main and rerun its exact-head coverage evidence** diff --git a/docs/superpowers/plans/2026-08-16-durable-provider-catalog.md b/docs/superpowers/plans/2026-08-16-durable-provider-catalog.md new file mode 100644 index 000000000..39069365d --- /dev/null +++ b/docs/superpowers/plans/2026-08-16-durable-provider-catalog.md @@ -0,0 +1,276 @@ +# Durable Provider Catalog Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Persist the five organization provider credentials and normalized model catalogs, then start `contextual-orchestrator` from an automatically discovered, role-tagged, multi-provider agent pool. + +**Architecture:** A trusted default-branch workflow seeds the existing encrypted credential registry and refreshes a normalized PostgreSQL provider catalog account by account. Runtime startup loads enabled catalog rows into ordinary `ModelAgent` records and uses the existing route/conduct engine plus a narrow native Bytez transport. Failures remain provider-scoped when last-known-good data exists and fail closed when no usable candidate remains. + +**Tech Stack:** Python 3.10+, standard-library HTTP/TLS, PostgreSQL + pgcrypto/psycopg optional DB extra, pytest/Hypothesis-compatible deterministic tests, GitHub Actions. + +## Global Constraints + +- Runtime provider keys resolve from the credential registry, never directly from environment variables. +- GitHub Actions environment variables are bootstrap transport only. +- Fixed credentials: `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, `BYTEZ_API_KEY`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`. +- Production durable bootstrap also requires `CONTEXTUAL_ORCHESTRATOR_KV_DSN` and `CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE`. +- Database objects use two-or-more-word snake_case and third normal form. +- Capability and role fit outrank known price; price is a bounded tie-break. +- Provider catalog/network/database errors never include credential values or raw provider bodies. +- Exact-head repository coverage and public-docstring coverage remain 100%. +- Existing security, fuzz, review, and branch-protection gates may not be weakened. + +--- + +### Task 1: Lock the provider inventory and normalization contracts + +**Files:** +- Create: `tests/test_provider_catalog.py` +- Create: `tests/test_provider_catalog_coverage.py` +- Create: `contextual_orchestrator/provider_catalog.py` + +**Interfaces:** +- Produces: `ProviderAccount`, `DiscoveredModel`, `CatalogModelRecord`, `DEFAULT_PROVIDER_ACCOUNTS`, `normalize_models_document(document) -> list[DiscoveredModel]`. +- Consumes: `register_credential`, `get_credential`, `ModelAgent`. + +- [x] **Step 1: Write failing inventory and normalization tests** + +```python +def test_default_accounts_cover_every_configured_secret_and_split_nvidia_accounts(): + assert [row.credential_name for row in DEFAULT_PROVIDER_ACCOUNTS] == [ + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "BYTEZ_API_KEY", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", + ] +``` + +- [x] **Step 2: Verify RED** + +Run: + +```bash +python -m pytest tests/test_provider_catalog.py -q +``` + +Expected before implementation: import failure for `contextual_orchestrator.provider_catalog`. + +- [x] **Step 3: Implement the fixed accounts and provider-neutral normalizer** + +Implement bounded ids, contexts, prices, modalities, and conservative capability inference. Keep unknown values `None`; never fabricate a price or context window. + +- [x] **Step 4: Verify GREEN** + +```bash +python -m pytest tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py -q +``` + +Expected: inventory and normalization contracts pass. + +- [x] **Step 5: Commit** + +```bash +git add tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py contextual_orchestrator/provider_catalog.py +git commit -m "feat: add durable multi-provider model catalog" +``` + +### Task 2: Add isolated refresh and normalized persistence + +**Files:** +- Modify: `contextual_orchestrator/provider_catalog.py` +- Test: `tests/test_provider_catalog.py` +- Test: `tests/test_provider_catalog_coverage.py` + +**Interfaces:** +- Produces: `ProviderCatalogStore`, `InMemoryProviderCatalogStore`, `PostgresProviderCatalogStore`, `ProviderCatalogService.refresh_all()`, `PROVIDER_CATALOG_SCHEMA_SQL`. +- Consumes: provider accounts and normalized models from Task 1. + +- [x] **Step 1: Write failing last-known-good and no-candidate tests** + +```python +def test_refresh_isolates_provider_failure_and_preserves_last_known_good_catalog(): + # Seed two accounts; fail one refresh; assert its old model remains and peer updates. + ... +``` + +- [x] **Step 2: Verify RED** + +```bash +python -m pytest tests/test_provider_catalog.py -k refresh -q +``` + +Expected before implementation: missing service/store methods. + +- [x] **Step 3: Implement account-scoped transactions and refresh evidence** + +Use `provider_accounts`, `provider_models`, `model_capabilities`, `model_modalities`, and `catalog_refresh_runs`. Disable missing models only after a complete successful account refresh. A failed refresh inserts failure evidence and leaves prior model rows untouched. + +- [x] **Step 4: Verify GREEN and schema rules** + +```bash +python -m pytest tests/test_provider_catalog.py -k "refresh or schema" -q +``` + +Expected: isolated refresh, stale availability, empty-catalog failure, and no-secret schema tests pass. + +- [x] **Step 5: Commit** + +```bash +git add contextual_orchestrator/provider_catalog.py tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py +git commit -m "feat: persist normalized provider catalogs" +``` + +### Task 3: Build the automatic runtime pool and native Bytez seam + +**Files:** +- Modify: `contextual_orchestrator/provider_catalog.py` +- Modify: `contextual_orchestrator/__init__.py` +- Modify: `contextual_orchestrator/__main__.py` +- Create: `tests/test_provider_catalog_cli.py` +- Test: `tests/test_provider_catalog.py` + +**Interfaces:** +- Produces: `ProviderCatalogService.candidate_agents()`, `ProviderAwareModelClient`, `build_catalog_orchestrator()`, CLI `--provider-catalog-dsn`. +- Consumes: `TaskOrchestrator`, `ModelClient`, enabled catalog rows, KV credential names. + +- [x] **Step 1: Write failing role-routing, failover, Bytez, and CLI tests** + +```python +def test_catalog_orchestrator_uses_role_tags_and_retains_cross_provider_failover(): + orchestrator = build_catalog_orchestrator(store, accounts=(reasoning, coding)) + assert orchestrator._select_agent("plan", "thinker").model == "deep-reasoner" + assert orchestrator._select_agent("implement code", "worker").model == "code-specialist" +``` + +- [x] **Step 2: Verify RED** + +```bash +python -m pytest tests/test_provider_catalog.py tests/test_provider_catalog_cli.py -q +``` + +Expected before implementation: missing catalog factory/client/CLI option. + +- [x] **Step 3: Implement agent conversion and provider-aware transport** + +Generate bounded two-or-more-word snake-case ids. Map chat/reasoning/coding/vision/audio capabilities into existing role tags. Keep role fit ahead of context and price. Delegate OpenAI-compatible providers to the existing secure client; use native Bytez `Key` plus `input` only for ordinary Bytez chat and fail closed for unsupported passthrough shapes. + +- [x] **Step 4: Verify GREEN** + +```bash +python -m pytest tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py tests/test_provider_catalog_cli.py -q +``` + +Expected: role selection, two-account NIM failover, Bytez native output, and catalog CLI startup pass. + +- [x] **Step 5: Commit** + +```bash +git add contextual_orchestrator/provider_catalog.py contextual_orchestrator/__init__.py contextual_orchestrator/__main__.py tests/test_provider_catalog*.py +git commit -m "feat: start runtime from discovered provider models" +``` + +### Task 4: Add the trust-separated GitHub Actions bootstrap + +**Files:** +- Create: `.github/workflows/provider-catalog-sync.yml` +- Test: `tests/test_provider_catalog.py` + +**Interfaces:** +- Produces: pull-request offline contract job and protected-main credential/catalog synchronization job. +- Consumes: fixed provider secrets, durable KV DSN/passphrase, module CLI `bootstrap-and-sync`. + +- [x] **Step 1: Encode the untrusted/trusted job boundary** + +Pull requests receive no provider or database secrets. Scheduled/manual execution is restricted to `refs/heads/main` and the protected `production` environment. + +- [x] **Step 2: Add complete-inventory validation** + +```bash +required=( + CONTEXTUAL_ORCHESTRATOR_KV_DSN + CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE + NVIDIA_NIM_API_KEY NVIDIA_NIM_API_KEY_SUB BYTEZ_API_KEY OPENROUTER_API_KEY OPENAI_API_KEY +) +``` + +Fail before bootstrap when any value is empty; add each configured value to Actions masking without printing it. + +- [x] **Step 3: Seed, refresh, and verify secret-free evidence** + +```bash +python -m contextual_orchestrator.provider_catalog bootstrap-and-sync \ + --require-all --agents-output "$RUNNER_TEMP/provider-agents.json" +``` + +Parse the generated agent pool and safe summary; fail if no candidate exists or any secret value appears. + +- [x] **Step 4: Validate workflow syntax and offline contracts** + +```bash +python -m pytest tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py -q +python -m compileall -q contextual_orchestrator +``` + +- [x] **Step 5: Commit** + +```bash +git add .github/workflows/provider-catalog-sync.yml +git commit -m "ci: add trusted provider catalog bootstrap" +``` + +### Task 5: Ground, document, and verify the exact head + +**Files:** +- Create: `docs/superpowers/specs/2026-08-16-durable-provider-catalog-design.md` +- Create: `docs/superpowers/plans/2026-08-16-durable-provider-catalog.md` +- Create: `docs/doctoring/durable-provider-catalog.md` +- Create: `docs/provider_catalog.md` +- Modify: `CHANGELOG.md` + +**Interfaces:** +- Produces: operator recovery/rollback instructions, APA 7 source record, release note. +- Consumes: implementation and workflow behavior from Tasks 1–4. + +- [x] **Step 1: Document architecture and operational actions** + +State credential/catalog separation, 3NF objects, refresh semantics, Bytez native boundary, startup behavior, exact secret inventory, and required DB bootstrap secrets. + +- [x] **Step 2: Add APA 7 doctoring** + +Ground the design in FrugalGPT, RouteLLM, Fugu/TRINITY/Conductor, HTTP semantics, NIST AI RMF, and PostgreSQL pgcrypto. Do not attach a PDF unless redistribution is permitted; reuse the repository's existing OA routing PDFs. + +- [ ] **Step 3: Run exact full verification** + +```bash +python -m coverage erase +python -m coverage run --branch -m pytest -q +python -m coverage report --fail-under=100 +interrogate --fail-under 100 contextual_orchestrator +python -m compileall -q contextual_orchestrator +python -m pip check +git diff --check +``` + +Expected: zero failures, 100% measured branch coverage, 100% public-docstring coverage, no dependency conflict, and no whitespace errors. + +- [ ] **Step 4: Run repository security gates** + +```bash +trivy --download-db-only +trivy fs --severity CRITICAL,HIGH --ignore-unfixed . +python -m pip_audit -r requirements.lock +``` + +Expected: no unremediated high/critical finding. Do not weaken a gate. + +- [ ] **Step 5: Publish the stacked PR and wait for protected evidence** + +```bash +git push -u origin feature/durable-provider-catalog-v2 +gh pr create --base fix/atheris-interpreter-lock \ + --head feature/durable-provider-catalog-v2 \ + --title "feat: durable automatic multi-provider catalog" +``` + +Target the accepted provider-security branch so DNS-pinned/strict-response work is inherited before protected `main`. Require all exact-head checks, current reviews, zero unresolved valid findings, and qualifying independent approval before normal merge. diff --git a/docs/superpowers/specs/2026-08-05-atheris-interpreter-lock-design.md b/docs/superpowers/specs/2026-08-05-atheris-interpreter-lock-design.md new file mode 100644 index 000000000..ea82d2646 --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-atheris-interpreter-lock-design.md @@ -0,0 +1,58 @@ +# Interpreter-Portable Atheris Lock Design + +## Status + +Approved for autonomous implementation under issue #95. This is a narrowly scoped prerequisite for the repository's same-head coverage and independent-review gates. + +## Problem + +The repository fuzz job and central coverage-evidence job use different supported CPython minor versions. A single unconditional Atheris pin makes one environment install a release that is not the intended artifact for that interpreter, which prevents coverage evidence from being produced and leaves otherwise repaired security work unmergeable. + +## Decision + +Use standardized Python dependency environment markers in both project metadata and the hash-locked fuzz requirements: + +- CPython below 3.13 selects `atheris==3.0.0`. +- CPython 3.13 and later selects `atheris==3.1.0`. +- Every eligible distribution remains protected by an explicit SHA-256 hash. +- One universal lock remains the source of truth for all supported runners. + +No runtime module, provider transport, reviewer identity, secret name, workflow permission, or database object changes. + +## Components + +### Project metadata + +`pyproject.toml` declares both mutually exclusive requirements inside the existing `fuzz` extra. The markers are part of the standardized dependency-specifier language and are evaluated by installation tooling for the active environment. + +### Universal hash lock + +`fuzz/requirements-atheris.in` records why the interpreter split exists. `fuzz/requirements-atheris.txt` carries the same mutually exclusive markers and the published SHA-256 hashes for the selected Atheris artifacts. + +### Contract test + +`tests/test_fuzz_dependency_lock.py` reads the metadata and lock as data. It proves: + +1. Python 3.11 selects exactly Atheris 3.0.0. +2. Python 3.13 and 3.14 select exactly Atheris 3.1.0. +3. The project markers and lock markers describe the same partition. +4. Every Atheris lock entry has at least one SHA-256 hash and the expected published hashes are present. +5. No interpreter selects zero or multiple Atheris releases. + +The test performs no network access and does not import Atheris. + +## Failure semantics + +Malformed, overlapping, incomplete, or unhashed requirements fail the normal test suite. Installation remains fail-closed through `--require-hashes`; this change does not introduce an unhashed fallback. + +## Documentation and release evidence + +`docs/doctoring/atheris-interpreter-lock.md` records the packaging specification and artifact evidence with APA 7 references. `CHANGELOG.md` records the compatibility change under `Unreleased`. + +## Non-goals + +- changing fuzz targets or fuzz budgets; +- changing provider egress or security behavior; +- changing the existing OpenCode review-agent credential scheme; +- changing scheduled workflows; +- upgrading unrelated dependencies. diff --git a/docs/superpowers/specs/2026-08-16-durable-provider-catalog-design.md b/docs/superpowers/specs/2026-08-16-durable-provider-catalog-design.md new file mode 100644 index 000000000..24c0cf501 --- /dev/null +++ b/docs/superpowers/specs/2026-08-16-durable-provider-catalog-design.md @@ -0,0 +1,187 @@ +# Durable Provider Catalog Design + +## Decision + +`contextual-orchestrator` will treat provider credentials, provider accounts, +model metadata, and orchestration policy as separate control-plane objects. +GitHub Actions secrets are bootstrap transport only. A trusted default-branch +workflow writes the five provider credentials into the existing pgcrypto-backed +credential registry, discovers each account's current model catalog, and stores +normalized model metadata in PostgreSQL. The running gateway reads credential +*names* and model candidates from those durable stores; it does not use provider +API-key environment variables as a runtime source. + +The fixed bootstrap inventory is: + +| Provider account | Credential name | Discovery/transport | +| --- | --- | --- | +| `nvidia_nim_primary` | `NVIDIA_NIM_API_KEY` | OpenAI-compatible `/v1/models` and chat | +| `nvidia_nim_secondary` | `NVIDIA_NIM_API_KEY_SUB` | Independent NIM account, same contract | +| `bytez_primary` | `BYTEZ_API_KEY` | Native Bytez `Key` and `input` contract | +| `openrouter_primary` | `OPENROUTER_API_KEY` | OpenAI-compatible model catalog and chat | +| `openai_primary` | `OPENAI_API_KEY` | OpenAI model catalog and chat | + +NVIDIA's primary and secondary keys remain distinct provider accounts so +quota exhaustion, revocation, health, and circuit state cannot be conflated. + +## Product outcome + +An operator configures the database connection and the five existing Actions +secrets once. The trusted sync job then maintains a candidate pool without +hand-editing an agents JSON file. At service startup, +`--provider-catalog-dsn` replaces the seed file with enabled database models. +The existing paper-grounded route/conduct engine receives the whole role-tagged +pool and continues to decide between one-model routing and a +Thinker–Worker–Verifier–Synthesizer workflow. + +The design does not claim that every listed model is suitable for every task. +Capabilities and modalities constrain routing first; context capacity and +provider/account preference follow; known price is only a small tie-break. The +current deterministic policy remains auditable and replaceable by a learned +router only after evaluation evidence shows that it is the bottleneck. + +## Boundaries + +### Credential plane + +The existing `provider_credentials` table remains the only provider-secret +store. It contains `credential_name` and pgcrypto-encrypted values. The catalog +stores only `credential_name` references. Secret values never appear in model +rows, generated agent JSON, audit summaries, workflow artifacts, or error text. + +`bootstrap_provider_credentials()` validates the complete fixed inventory before +writing when `--require-all` is selected. This prevents a production run from +rotating only a subset and leaving an ambiguous mixed generation. + +### Catalog plane + +The catalog is third-normal-form data: + +- `provider_accounts`: account identity, provider, credential name, endpoint, + transport, enablement, and priority; +- `provider_models`: account-specific model identity, display metadata, context, + known prices, enablement, and first/last observation; +- `model_capabilities`: one capability per model row; +- `model_modalities`: one modality per model row; +- `catalog_refresh_runs`: immutable per-account refresh outcome evidence. + +A successful account refresh atomically upserts the observed set and disables +models absent from that complete response. A failed refresh writes only a +failure record; it never disables or deletes the prior usable set. + +### Discovery plane + +Credentialed catalog HTTP uses HTTPS, direct DNS-resolved public addresses, +normal certificate/SNI verification, no redirect following, no ambient proxy, +a bounded response, strict JSON object validation, bounded attempts, jittered +backoff, and a wall-clock deadline. Authentication and schema errors fail fast. +Transient network, rate-limit, and 5xx errors are isolated to that account. + +The model normalizer accepts the common `data` and `models` shapes, rejects +invalid/oversized identifiers and non-finite metadata, and infers conservative +capabilities from provider metadata plus model naming. Unknown values remain +unknown; the gateway does not fabricate context windows or prices. + +### Inference plane + +OpenAI, OpenRouter, and NVIDIA NIM continue through the hardened +OpenAI-compatible `ModelClient`. Bytez uses `ProviderAwareModelClient` and its +native `Authorization: Key …` plus `{"input": …}` request shape. Unsupported +Bytez passthrough endpoints fail closed instead of pretending that a native +response is an OpenAI Responses or tool-call object. + +Generated `ModelAgent` rows contain model ids, endpoints, provider names, +capabilities, priorities, and credential names only. `TaskOrchestrator` retains +its existing per-agent retry, failover, and circuit-breaker behavior. A provider +catalog with zero enabled candidates is a startup error, not a reason to start a +mock agent. + +## Trusted GitHub Actions flow + +`.github/workflows/provider-catalog-sync.yml` has two trust-separated jobs: + +1. Pull requests run only deterministic offline contracts and compile checks; + provider secrets are not exposed to contributor code. +2. Scheduled/manual runs execute only on protected `main` in the `production` + environment. They require the five provider keys plus + `CONTEXTUAL_ORCHESTRATOR_KV_DSN` and + `CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE`, seed the encrypted registry, refresh + metadata, and verify that generated agent evidence contains no secret value. + +Missing database bootstrap secrets block the job. The workflow never downgrades +to process memory, because an ephemeral registry would create a false success +and disappear before the service could use it. + +## Failure semantics + +| Condition | Result | Operator action | +| --- | --- | --- | +| One provider catalog is unavailable and has prior models | Serve prior models as `stale_available`; refresh peers | Inspect provider health; retry next schedule | +| One provider is unavailable with no prior models | Mark account `failed`; continue peers | Correct endpoint/key or wait for provider | +| All providers fail and no prior model exists | Fail sync/startup | Restore DB/provider connectivity before service | +| Credential missing | Account failure; production `--require-all` blocks before writes | Add/repair the named Actions secret | +| 401/403 | Permanent account failure, no retry storm | Rotate/re-authorize that credential | +| 408/429/5xx/network timeout | Bounded jittered retry, then stale/failure classification | Observe rate and provider SLO | +| Invalid/oversized/non-JSON response | Fail closed without body disclosure | Treat as provider contract/security incident | +| Database failure | Fail closed; no memory fallback | Restore the authoritative catalog/KV database | +| Bytez unsupported response/passthrough | Fail closed, allow normal orchestrator failover where available | Use a supported native chat model or another provider | + +## Test and acceptance evidence + +The feature is accepted only when all of the following hold on one exact PR +head: + +- all five fixed credential names are represented and NVIDIA accounts remain + independent; +- required bootstrap is all-or-nothing and summaries contain no value; +- normalization, malformed metadata, specialized capabilities, and price/context + bounds are deterministic; +- provider failures are isolated and last-known-good models survive; +- no-candidate startup fails closed; +- discovered models become valid two-or-more-word snake-case agents; +- role selection and cross-provider failover use the complete candidate pool; +- native Bytez authentication/response handling is tested independently; +- PostgreSQL DDL is normalized and contains no secret-value column; +- the full repository test, 100% branch coverage, 100% public docstring, + security, fuzz, and protected review gates pass without weakening them; and +- the protected default-branch sync subsequently records real provider and DB + evidence without revealing credentials. + +## Research and standards basis + +The route/conduct split follows the repository's existing Fugu, TRINITY, and +Conductor interpretation: cheap single-model selection for suitable work, deeper +role-separated computation when decomposition and verification add value. The +catalog makes the swappable model pool operational rather than static. Cost is +kept subordinate to capability, consistent with cost-aware routing literature +that optimizes under quality constraints rather than choosing the cheapest model +unconditionally. + +### References + +Chen, L., Zaharia, M., & Zou, J. (2023). *FrugalGPT: How to use large language +models while reducing cost and improving performance*. arXiv. +https://doi.org/10.48550/arXiv.2305.05176 + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). +Internet Engineering Task Force. https://doi.org/10.17487/RFC9110 + +*Learning to orchestrate agents in natural language with the Conductor*. +(2025). arXiv. https://arxiv.org/abs/2512.04388 + +National Institute of Standards and Technology. (2023). *Artificial intelligence +risk management framework (AI RMF 1.0)* (NIST AI 100-1). +https://doi.org/10.6028/NIST.AI.100-1 + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., Kadous, +M. W., & Stoica, I. (2024). *RouteLLM: Learning to route LLMs with preference +data*. arXiv. https://doi.org/10.48550/arXiv.2406.18665 + +PostgreSQL Global Development Group. (2026). *pgcrypto*. +https://www.postgresql.org/docs/current/pgcrypto.html + +Sakana AI. (2026). *Fugu technical report*. +https://github.com/SakanaAI/fugu/blob/main/Fugu_technical_report.pdf + +*TRINITY: An evolved LLM coordinator*. (2025). arXiv. +https://arxiv.org/abs/2512.04695 diff --git a/fuzz/requirements-atheris.in b/fuzz/requirements-atheris.in index b930f7524..f90eaaec8 100644 --- a/fuzz/requirements-atheris.in +++ b/fuzz/requirements-atheris.in @@ -1,3 +1,9 @@ -# Atheris coverage-guided job deps (Python 3.11). Compile: uv pip compile fuzz/requirements-atheris.in --generate-hashes --python-version 3.11 --universal -o fuzz/requirements-atheris.txt +# Atheris coverage-guided job deps. atheris is published per-interpreter: the +# repo fuzz job runs CPython 3.11, where the newest published wheel is 3.0.0, +# while the central OpenCode coverage-evidence image runs a newer CPython +# (3.13+) where only 3.1.0 is published. Pin per interpreter with environment +# markers so a single hash lock satisfies both --require-hashes installs. +# Compile: uv pip compile fuzz/requirements-atheris.in --generate-hashes --python-version 3.11 --universal -o fuzz/requirements-atheris.txt pip -atheris==3.0.0 +atheris==3.0.0; python_version < "3.13" +atheris==3.1.0; python_version >= "3.13" diff --git a/fuzz/requirements-atheris.txt b/fuzz/requirements-atheris.txt index b3e913ba6..49919d9c3 100644 --- a/fuzz/requirements-atheris.txt +++ b/fuzz/requirements-atheris.txt @@ -1,11 +1,16 @@ # This file was autogenerated by uv via the following command: # uv pip compile fuzz/requirements-atheris.in --generate-hashes --python-version 3.11 --universal -o fuzz/requirements-atheris.txt -atheris==3.0.0 \ +atheris==3.0.0 ; python_full_version < '3.13' \ --hash=sha256:1f0929c7bc3040f3fe4102e557718734190cf2d7718bbb8e3ce6d3eb56ef5bb3 \ --hash=sha256:510e502c57b6dc615fb174066407af620d4c7f73cf08a782c86e7761bf12c4eb \ --hash=sha256:8a5c8a781467c187da40fd29139784193e2647058831f837f675d0bb8cbd8746 \ --hash=sha256:a402cdca8a650d1371050b1f9552eb4cdc488d2db64950d603c4560318365eac # via -r fuzz/requirements-atheris.in +atheris==3.1.0 ; python_full_version >= '3.13' \ + --hash=sha256:315a0b5c819852b1ffe1ca72efc389c7724881f2c33e4aacb8c6bcec49bd5011 \ + --hash=sha256:ec5e11f21a4c197fe91f7aea2b2de88e623c73a21fc07b105ac6329a1588457b \ + --hash=sha256:f8a9f51ce8369026e8eb7b7174835e8c4c85a1a6db5d9add36c15100779d2a39 + # via -r fuzz/requirements-atheris.in pip==26.1.2 \ --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ --hash=sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605 diff --git a/pyproject.toml b/pyproject.toml index 65bd69eac..45e774326 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,24 @@ +[build-system] +requires = ["setuptools==83.0.0"] +build-backend = "setuptools.build_meta" + [project] name = "contextual-orchestrator" version = "0.1.0" -description = "Paper-grounded model orchestration lab with an enterprise admin console." +description = "Provider-neutral OpenAI-compatible orchestration control plane for governed routing and multi-agent conduct." readme = "README.md" requires-python = ">=3.10" +license = "MIT" +license-files = ["LICENSE"] dependencies = [ "hypothesis>=6.100", ] +[project.urls] +Homepage = "https://github.com/ContextualWisdomLab/contextual-orchestrator" +Repository = "https://github.com/ContextualWisdomLab/contextual-orchestrator" +Issues = "https://github.com/ContextualWisdomLab/contextual-orchestrator/issues" + [project.optional-dependencies] api = [ "fastapi>=0.128.0", @@ -20,6 +31,7 @@ db = [ ] fuzz = [ "atheris==3.0.0; python_version < '3.13'", + "atheris==3.1.0; python_version >= '3.13'", ] [tool.contextual_orchestrator] @@ -30,14 +42,15 @@ include = ["contextual_orchestrator*"] [tool.coverage.run] source = ["contextual_orchestrator"] -omit = ["contextual_orchestrator/__main__.py", "contextual_orchestrator/server.py"] +branch = true [tool.coverage.report] show_missing = true +fail_under = 100 [tool.interrogate] exclude = ["tests"] -fail-under = 80 +fail-under = 100 ignore-init-method = true ignore-magic = true ignore-nested-functions = true diff --git a/tests/test_batch_ledger_router_coverage.py b/tests/test_batch_ledger_router_coverage.py new file mode 100644 index 000000000..58d0e96af --- /dev/null +++ b/tests/test_batch_ledger_router_coverage.py @@ -0,0 +1,544 @@ +"""Behavioural coverage for batch routing, the cost ledger, and the router. + +Covers the embeddings pg-llm-batch backend (driven by a fake async client that +mirrors ``BatchAPIClient``), batch/embedding edge cases, the non-blocking ledger +store's queue-full / flush-timeout / stored paths, SQL window queries, and the +cost-routing coordinator's split/token/embedding-document edge behaviours. Every +test asserts the real result, not just line execution. +""" + +from __future__ import annotations + +from pathlib import Path +import sqlite3 +import sys +import threading + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.batch_routing import ( # noqa: E402 + BatchJob, + EmbeddingBatchRequest, + EmbeddingBatchResultItem, + LocalEmbeddingBatchBackend, + PgLlmBatchEmbeddingBackend, + _extract_answer, + _extract_embedding, + build_embeddings_jsonl_body, + cheapest_upstream, + heuristic_embedding, +) +from contextual_orchestrator.cost_ledger import ( # noqa: E402 + ATTRIBUTION_DIMENSION_CATALOG, + AttributionDimensions, + CostLedger, + InMemoryLedgerStore, + InMemoryUsageTelemetrySink, + NonBlockingLedgerStore, + PriceBook, + PriceEntry, + SqlLedgerStore, + UsageRecord, + _emit_usage_event, +) +from contextual_orchestrator.cost_router import ( # noqa: E402 + CostRoutingCoordinator, + _positive_int, + _provider_from_base_url, + _weighted_average_embedding, +) +from contextual_orchestrator.kv_config import InMemoryConfigStore # noqa: E402 +from contextual_orchestrator.token_counting import HeuristicTokenCounter # noqa: E402 + + +# --------------------------------------------------------------------------- +# batch_routing: small edge helpers +# --------------------------------------------------------------------------- + + +def test_cheapest_upstream_returns_none_for_empty_candidates() -> None: + assert cheapest_upstream([], price_book=None) is None + + +def test_cheapest_upstream_keeps_scanning_after_a_more_expensive_candidate() -> None: + """A costly middle candidate cannot stop a later cheaper upstream winning.""" + price_book = PriceBook(InMemoryConfigStore()) + price_book.set_price(PriceEntry("first", "model", 2.0, 2.0)) + price_book.set_price(PriceEntry("costly", "model", 4.0, 4.0)) + price_book.set_price(PriceEntry("cheapest", "model", 1.0, 1.0)) + candidates = [ + {"provider": "first", "model": "model"}, + {"provider": "costly", "model": "model"}, + {"provider": "cheapest", "model": "model"}, + ] + + assert cheapest_upstream(candidates, price_book) == candidates[2] + + +def test_extract_answer_and_embedding_handle_empty_bodies() -> None: + assert _extract_answer({}) == "" + assert _extract_answer({"choices": []}) == "" + assert _extract_embedding({}) == [] + assert _extract_embedding({"data": [{"embedding": [1, 2]}]}) == [1.0, 2.0] + + +def test_heuristic_embedding_rejects_non_positive_dimension() -> None: + with pytest.raises(ValueError): + heuristic_embedding("text", dimension=0) + assert len(heuristic_embedding("text", dimension=4)) == 4 + + +def test_local_embedding_backend_counts_tokens_without_counter() -> None: + backend = LocalEmbeddingBatchBackend() + job = backend.submit([EmbeddingBatchRequest(input_text="one two three", custom_id="e0")]) + item = backend.retrieve(job)[0] + assert item.prompt_tokens == 3 + + +def test_build_embeddings_jsonl_and_to_jsonl_line_use_embeddings_endpoint() -> None: + body = build_embeddings_jsonl_body([EmbeddingBatchRequest(input_text="hi", custom_id="e1", model="emb-x")]) + assert '"url": "/v1/embeddings"' in body + assert '"custom_id": "e1"' in body + assert '"input": "hi"' in body + + +class _FakeEmbeddingApiClient: + """Mimics pg_llm_batch.BatchAPIClient's async surface for the embeddings path.""" + + def __init__(self) -> None: + self.calls: list[str] = [] + self.created_endpoint: str | None = None + self.last_metadata: dict | None = None + + async def upload_jsonl(self, file_path, endpoint_alias, purpose="batch"): + self.calls.append("upload_jsonl") + return {"id": "file-emb"} + + async def create_batch_job(self, input_file_id, endpoint_alias, endpoint="/v1/embeddings", metadata=None): + self.calls.append("create_batch_job") + assert input_file_id == "file-emb" + self.created_endpoint = endpoint + self.last_metadata = metadata + return {"id": "batch-emb", "status": "validating"} + + async def get_batch_status(self, batch_id, endpoint_alias): + self.calls.append("get_batch_status") + return {"status": "completed", "is_complete": True, "progress_percentage": 100} + + async def download_results(self, batch_id, endpoint_alias): + self.calls.append("download_results") + return { + "success": True, + "responses": [ + {"custom_id": "b", "response": {"body": {"data": [{"embedding": [0.3, 0.4]}], "usage": {"prompt_tokens": 5}}}}, + {"custom_id": "a", "response": {"body": {"data": [{"embedding": [0.1, 0.2]}], "usage": {"prompt_tokens": 3}}}}, + ], + } + + +def test_pg_llm_batch_embedding_backend_submits_polls_and_orders_results() -> None: + client = _FakeEmbeddingApiClient() + backend = PgLlmBatchEmbeddingBackend(client, endpoint_alias="prod_gateway") + requests = [ + EmbeddingBatchRequest(input_text="alpha", custom_id="a", model="emb-x"), + EmbeddingBatchRequest(input_text="beta", custom_id="b", model="emb-x"), + ] + + job = backend.submit(requests, metadata={"routing_reason": "bulk"}) + assert job.backend == "pg-llm-batch" + assert job.job_id == "batch-emb" + assert client.created_endpoint == "/v1/embeddings" + assert client.last_metadata == {"routing_reason": "bulk"} + assert backend.poll(job)["is_complete"] is True + + items = backend.retrieve(job) + assert [item.custom_id for item in items] == ["a", "b"] + assert [item.index for item in items] == [0, 1] + assert items[0].embedding == [0.1, 0.2] + assert items[0].prompt_tokens == 3 + assert items[1].model == "emb-x" + + +def test_pg_llm_batch_embedding_backend_incomplete_download_returns_empty() -> None: + class _IncompleteClient(_FakeEmbeddingApiClient): + async def download_results(self, batch_id, endpoint_alias): + return {"success": False, "reason": "not complete"} + + backend = PgLlmBatchEmbeddingBackend(_IncompleteClient()) + job = backend.submit([EmbeddingBatchRequest(input_text="x", custom_id="a")]) + assert backend.retrieve(job) == [] + + +def test_pg_llm_batch_embedding_backend_uses_payload_assembler_when_present() -> None: + captured: dict = {} + + class _Assembler: + def assemble(self, lines): + captured["lines"] = lines + return "memory://assembled" + + backend = PgLlmBatchEmbeddingBackend(_FakeEmbeddingApiClient(), payload_assembler=_Assembler()) + backend.submit([EmbeddingBatchRequest(input_text="hi", custom_id="a")]) + assert captured["lines"][0]["url"] == "/v1/embeddings" + + +# --------------------------------------------------------------------------- +# cost_ledger +# --------------------------------------------------------------------------- + + +def _record(record_id: str, created_at: int = 1) -> UsageRecord: + return UsageRecord( + usage_record_id=record_id, + created_at=created_at, + workflow_run_id=None, + request_channel="sync", + route_mode=None, + provider_name="provider_name", + model_name="model_name", + prompt_tokens=1, + completion_tokens=1, + total_tokens=2, + cost_amount=0.0, + currency_code="USD", + attribution=AttributionDimensions.from_mapping({}), + ) + + +def test_in_memory_usage_telemetry_sink_evicts_oldest_past_cap() -> None: + from contextual_orchestrator.cost_ledger import UsageTelemetryEvent + + sink = InMemoryUsageTelemetrySink(max_events=2) + for index in range(4): + sink.emit_usage(UsageTelemetryEvent.from_record(_record(f"u{index}"), export_state="queued")) + kept = sink.events() + assert len(kept) == 2 + assert [event.attributes["contextual_orchestrator.usage_record_id"] for event in kept] == ["u2", "u3"] + + +def test_emit_usage_event_swallows_sink_failures() -> None: + from contextual_orchestrator.cost_ledger import UsageTelemetryEvent + + class _RaisingSink: + def emit_usage(self, event) -> None: + raise RuntimeError("sink down") + + assert _emit_usage_event(_RaisingSink(), UsageTelemetryEvent.from_record(_record("u0"), export_state="queued")) is None + + +def test_non_blocking_store_rejects_zero_queue_size() -> None: + with pytest.raises(ValueError): + NonBlockingLedgerStore(InMemoryLedgerStore(), queue_size=0) + + +def test_non_blocking_store_drops_on_full_queue_and_flush_reports_timeout_then_stores() -> None: + class _BlockingBackend: + def __init__(self) -> None: + self.started = threading.Event() + self.release = threading.Event() + self.rows: list[UsageRecord] = [] + + def append(self, record: UsageRecord) -> None: + self.started.set() + self.release.wait(timeout=5) + self.rows.append(record) + + def query(self, start=None, end=None): + return [row.as_dict() for row in self.rows] + + backend = _BlockingBackend() + sink = InMemoryUsageTelemetrySink() + store = NonBlockingLedgerStore(backend, queue_size=1, telemetry_sink=sink) + + store.append(_record("first")) + assert backend.started.wait(timeout=5) + store.append(_record("queued")) + store.append(_record("dropped")) + + assert store.flush(timeout=0.05) is False + health = store.telemetry_health() + assert health["records_dropped"] == 1 + assert health["records_accepted"] == 2 + + backend.release.set() + assert store.flush(timeout=2.0) is True + assert store.telemetry_health()["records_stored"] == 2 + assert len(store.query()) == 2 + dropped_events = [ + event + for event in sink.events() + if event.attributes["contextual_orchestrator.usage.export_state"] == "dropped" + ] + assert len(dropped_events) == 1 + + +def test_in_memory_ledger_store_len_reflects_appended_rows() -> None: + store = InMemoryLedgerStore() + assert len(store) == 0 + store.append(_record("u0")) + assert len(store) == 1 + + +def _priced_ledger(**kwargs) -> CostLedger: + price_book = PriceBook(InMemoryConfigStore()) + price_book.set_price(PriceEntry("openai", "gpt-x", prompt_price_per_1k=2.0, completion_price_per_1k=4.0)) + return CostLedger(price_book, **kwargs) + + +def test_cost_ledger_wraps_store_when_non_blocking_requested() -> None: + ledger = _priced_ledger(non_blocking_store=True) + assert isinstance(ledger.store, NonBlockingLedgerStore) + ledger.record_usage(provider="openai", model="gpt-x", prompt_tokens=10, completion_tokens=5) + assert ledger.flush(timeout=2.0) + assert len(ledger.records()) == 1 + + +def test_cost_ledger_accepts_prebuilt_attribution_dimensions() -> None: + ledger = _priced_ledger() + dims = AttributionDimensions.from_mapping({"team": "alpha", "company": "acme"}) + record = ledger.record_usage( + provider="openai", model="gpt-x", prompt_tokens=10, completion_tokens=5, attribution=dims + ) + row = record.as_dict() + assert row["team_name"] == "alpha" + assert row["company_name"] == "acme" + + +def test_cost_ledger_inline_store_failure_is_recorded_and_survives() -> None: + class _FailingStore: + def append(self, record) -> None: + raise RuntimeError("P2028 with secret prompt") + + def query(self, start=None, end=None): + return [] + + sink = InMemoryUsageTelemetrySink() + price_book = PriceBook(InMemoryConfigStore()) + price_book.set_price(PriceEntry("openai", "gpt-x", 2.0, 4.0)) + ledger = CostLedger(price_book, store=_FailingStore(), telemetry_sink=sink) + + record = ledger.record_usage(provider="openai", model="gpt-x", prompt_tokens=10, completion_tokens=5) + assert record.usage_record_id.startswith("usage_") + health = ledger.telemetry_health() + assert health["store_failures"] == 1 + assert health["records_accepted"] == 1 + assert health["last_error_type"] == "RuntimeError" + assert "P2028" not in repr(sink.events()) + + +def test_cost_ledger_flush_is_noop_true_for_plain_store() -> None: + assert _priced_ledger().flush() is True + + +def test_sql_ledger_store_query_honours_time_window() -> None: + conn = sqlite3.connect(":memory:") + store = SqlLedgerStore(conn, paramstyle="qmark") + ledger = _priced_ledger(store=store) + ledger.record_usage(provider="openai", model="gpt-x", prompt_tokens=10, completion_tokens=0, created_at=100) + ledger.record_usage(provider="openai", model="gpt-x", prompt_tokens=10, completion_tokens=0, created_at=300) + assert len(store.query(150, 400)) == 1 + assert len(store.query(None, 200)) == 1 + assert len(store.query(200, None)) == 1 + assert len(store.query()) == 2 + + +def test_sql_ledger_store_dimension_seed_is_idempotent() -> None: + """Reopening one ledger connection does not duplicate attribution dimensions.""" + conn = sqlite3.connect(":memory:") + SqlLedgerStore(conn) + SqlLedgerStore(conn) + + count = conn.execute("SELECT COUNT(*) FROM cost_attribution_dimensions").fetchone()[0] + assert count == len(ATTRIBUTION_DIMENSION_CATALOG) + + +def test_cost_ledger_ignores_falsey_text_health_and_merges_later_counters() -> None: + """A blank store-health detail cannot hide later numeric health counters.""" + + class _HealthStore(InMemoryLedgerStore): + def telemetry_health(self): + return {"last_error_type": "", "records_stored": 2} + + ledger = _priced_ledger(store=_HealthStore()) + health = ledger.telemetry_health() + + assert health["last_error_type"] is None + assert health["records_stored"] == 2 + + +# --------------------------------------------------------------------------- +# cost_router: pure helpers + coordinator edge behaviours +# --------------------------------------------------------------------------- + + +def test_provider_from_base_url_maps_mock_and_remote_hosts() -> None: + assert _provider_from_base_url("mock://local") == "mock" + assert _provider_from_base_url("https://api.openai.com/v1") == "api.openai.com" + + +def test_provider_from_base_url_fails_closed_when_parser_rejects_input( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A URL parser failure produces no guessed provider authority.""" + import urllib.parse + + monkeypatch.setattr( + urllib.parse, + "urlparse", + lambda _value: (_ for _ in ()).throw(ValueError("bad URL")), + ) + assert _provider_from_base_url("https://invalid.example") == "" + + +def test_positive_int_falls_back_on_invalid_or_non_positive() -> None: + assert _positive_int("not-a-number", 5) == 5 + assert _positive_int(None, 5) == 5 + assert _positive_int(-3, 5) == 5 + assert _positive_int(12, 5) == 12 + + +def test_weighted_average_embedding_empty_and_weighted() -> None: + assert _weighted_average_embedding([]) == [] + assert _weighted_average_embedding([([], 1)]) == [] + assert _weighted_average_embedding([([2.0], 1), ([4.0], 3)]) == [3.5] + + +def _coordinator(**kwargs) -> CostRoutingCoordinator: + orchestrator = TaskOrchestrator( + [ModelAgent("mock_worker", "mock-a", "mock://a", provider_name="mock", tags=("reasoning",), priority=1)] + ) + return CostRoutingCoordinator(orchestrator, InMemoryConfigStore(), **kwargs) + + +def test_served_provider_model_falls_back_when_agent_unresolvable() -> None: + coordinator = _coordinator() + provider, model = coordinator._served_provider_model( + {"trace": [{"served_agent_id": "ghost_agent"}]}, "fallback-model" + ) + assert (provider, model) == ("unknown", "fallback-model") + + +def test_served_provider_model_falls_back_when_trace_has_no_agent_identity() -> None: + """A trace row without an agent identity retains the caller's model fallback.""" + coordinator = _coordinator() + assert coordinator._served_provider_model({"trace": [{}]}, "fallback-model") == ( + "unknown", + "fallback-model", + ) + + +def test_batch_and_embedding_job_lookups_raise_for_unknown_ids() -> None: + coordinator = _coordinator() + with pytest.raises(KeyError): + coordinator.poll_batch("missing_job") + with pytest.raises(KeyError): + coordinator.embeddings_batch_document("missing_job") + + +def test_force_token_safe_chunks_empty_and_single_unit_midpoint_split() -> None: + coordinator = _coordinator() + assert coordinator._split_embedding_input("", model="m", max_tokens=10, max_chars=10) == [("", 0)] + assert coordinator._force_token_safe_chunks("", model="m", max_tokens=10, max_chars=10) == [("", 0)] + chunks = coordinator._force_token_safe_chunks("abcdefgh", model="m", max_tokens=1, max_chars=100) + assert "".join(text for text, _ in chunks) == "abcdefgh" + assert len(chunks) > 1 + + +def test_split_embedding_input_falls_back_when_adapter_counts_change() -> None: + """An inconsistent token adapter still yields bounded parts that preserve all text.""" + + class _SequenceCounter: + def __init__(self) -> None: + self.counts = iter([2, 1, 1, 1, 1, 1, 1]) + + def count_text(self, text, model): + return next(self.counts, 1) + + def count_messages(self, messages, model): + return 1 + + coordinator = _coordinator(token_counter=_SequenceCounter()) + chunks = coordinator._split_embedding_input("a b", model="m", max_tokens=1, max_chars=100) + + assert "".join(text for text, _tokens in chunks) == "a b" + assert len(chunks) == 2 + + +def test_count_embedding_tokens_tolerates_counter_failure_and_zero() -> None: + coordinator = _coordinator() + assert coordinator._count_embedding_tokens(" ", "m") == 1 + + class _RaisingCounter: + def count_text(self, text, model): + raise RuntimeError("boom") + + def count_messages(self, messages, model): + return 0 + + failing = _coordinator(token_counter=_RaisingCounter()) + assert failing._count_embedding_tokens("hi there", "m") == 2 + + +def test_embeddings_batch_document_returns_pending_envelope_when_incomplete() -> None: + class _PendingBackend: + name = "pending" + + def submit(self, requests, metadata=None): + self.requests = list(requests) + return BatchJob(job_id="emb-pending", backend=self.name, status="validating", request_count=len(self.requests)) + + def poll(self, job): + return {"status": "validating", "is_complete": False} + + def retrieve(self, job): + return [] + + coordinator = _coordinator(embedding_batch_backend=_PendingBackend()) + job = coordinator.submit_embeddings_batch(["alpha"], attribution={"provider": "acme"}) + document = coordinator.embeddings_batch_document(job.job_id) + assert document["status"] == "validating" + assert document["embeddings"] is None + assert coordinator.ledger.records() == [] + + +def test_embeddings_batch_document_handles_missing_source_parts_and_zero_token_items() -> None: + class _PartialBackend: + name = "partial" + + def submit(self, requests, metadata=None): + self.requests = list(requests) + return BatchJob(job_id="emb-partial", backend=self.name, status="completed", request_count=len(self.requests)) + + def poll(self, job): + return {"status": "completed", "is_complete": True} + + def retrieve(self, job): + first = self.requests[0] + return [ + EmbeddingBatchResultItem( + custom_id=first.custom_id, index=0, embedding=[1.0], prompt_tokens=0, model=first.model + ) + ] + + coordinator = _coordinator(embedding_batch_backend=_PartialBackend()) + document = coordinator.complete_embeddings_batch( + ["alpha", "beta"], model="emb-x", attribution={"provider": "acme"} + ) + assert [item["index"] for item in document["embeddings"]] == [0, 1] + assert document["embeddings"][1]["embedding"] == [] + assert document["token_counts"][1] == 0 + assert document["token_counts"][0] > 0 + + +if __name__ == "__main__": # pragma: no cover + import types + + for _name, _fn in sorted(globals().items()): + if _name.startswith("test_") and isinstance(_fn, types.FunctionType): + if _fn.__code__.co_argcount == 0: + _fn() + print(f"ok {_name}") + print("ok") diff --git a/tests/test_batch_routing_embeddings.py b/tests/test_batch_routing_embeddings.py new file mode 100644 index 000000000..9fea7db28 --- /dev/null +++ b/tests/test_batch_routing_embeddings.py @@ -0,0 +1,246 @@ +"""Embeddings batch routing: heuristic embedding, local + pg backends, helpers. + +Covers the offline embeddings path — ``heuristic_embedding``, the in-process +``LocalEmbeddingBatchBackend``, the ``PgLlmBatchEmbeddingBackend`` (via an async +fake client), and the module helpers — with no Postgres and no external service. +""" + +from __future__ import annotations + +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.batch_routing import ( # noqa: E402 + BatchJob, + BatchRequest, + EmbeddingBatchRequest, + LocalEmbeddingBatchBackend, + PgLlmBatchBackend, + PgLlmBatchEmbeddingBackend, + RoutingHints, + _extract_answer, + _extract_embedding, + build_embeddings_jsonl_body, + cheapest_upstream, + heuristic_embedding, +) + + +def test_cheapest_upstream_returns_none_for_no_candidates() -> None: + """Cheapest upstream returns none for no candidates.""" + assert cheapest_upstream([], None) is None + + +def test_extract_answer_empty_choices_is_blank() -> None: + """Extract answer empty choices is blank.""" + assert _extract_answer({"choices": []}) == "" + assert _extract_answer({}) == "" + + +def test_extract_embedding_reads_first_vector_or_empty() -> None: + """Extract embedding reads first vector or empty.""" + assert _extract_embedding({"data": [{"embedding": [0.1, 0.2, 0.3]}]}) == [0.1, 0.2, 0.3] + assert _extract_embedding({}) == [] + + +def test_heuristic_embedding_is_deterministic_and_ranged() -> None: + """Heuristic embedding is deterministic and ranged.""" + vector = heuristic_embedding("hello", dimension=8) + assert len(vector) == 8 + assert all(-1.0 <= value <= 1.0 for value in vector) + assert heuristic_embedding("hello", dimension=8) == vector + + +def test_heuristic_embedding_rejects_non_positive_dimension() -> None: + """Heuristic embedding rejects non positive dimension.""" + with pytest.raises(ValueError): + heuristic_embedding("hello", dimension=0) + + +def test_embedding_request_to_jsonl_line_shape() -> None: + """Embedding request to jsonl line shape.""" + line = EmbeddingBatchRequest(input_text="hi", model="embed-x", custom_id="e1").to_jsonl_line( + "/v1/embeddings" + ) + assert line["custom_id"] == "e1" + assert line["url"] == "/v1/embeddings" + assert line["body"] == {"model": "embed-x", "input": "hi"} + + +def test_build_embeddings_jsonl_body_is_newline_delimited() -> None: + """Build embeddings jsonl body is newline delimited.""" + body = build_embeddings_jsonl_body( + [ + EmbeddingBatchRequest(input_text="hi", model="embed-x", custom_id="e1"), + EmbeddingBatchRequest(input_text="yo", model="embed-x", custom_id="e2"), + ] + ) + assert body.count("\n") == 1 + assert '"custom_id": "e1"' in body + + +def test_local_embedding_backend_token_fallback_counts_words() -> None: + """Local embedding backend token fallback counts words.""" + backend = LocalEmbeddingBatchBackend(dimension=4) + job = backend.submit( + [EmbeddingBatchRequest(input_text="one two three", model="embed-x", custom_id="e1")] + ) + assert backend.poll(job)["is_complete"] is True + items = backend.retrieve(job) + assert len(items) == 1 + assert items[0].prompt_tokens == 3 + assert len(items[0].embedding) == 4 + + +class _FakeTokenCounter: + """Token counter returning a fixed count, exercising the counted path.""" + + def count_text(self, text: str, model: str) -> int: + """Return a constant token count regardless of input.""" + return 42 + + +def test_local_embedding_backend_uses_injected_token_counter() -> None: + """Local embedding backend uses injected token counter.""" + backend = LocalEmbeddingBatchBackend(token_counter=_FakeTokenCounter(), dimension=4) + job = backend.submit( + [EmbeddingBatchRequest(input_text="anything", model="embed-x", custom_id="e1")] + ) + assert backend.retrieve(job)[0].prompt_tokens == 42 + + +def test_local_embedding_backend_retrieve_unknown_job_is_empty() -> None: + """Local embedding backend retrieve unknown job is empty.""" + backend = LocalEmbeddingBatchBackend() + unknown = BatchJob(job_id="missing", backend="local", status="completed", request_count=0) + assert backend.retrieve(unknown) == [] + + +class _FakeEmbeddingClient: + """Async pg-llm-batch client fake returning one embedding response.""" + + def __init__(self) -> None: + """Start with an empty backend-call log.""" + self.calls: list[str] = [] + + async def upload_jsonl(self, file_path, endpoint_alias, purpose="batch"): + """Record the call and return a stub uploaded-file id.""" + self.calls.append("upload_jsonl") + return {"id": "file-emb"} + + async def create_batch_job( + self, input_file_id, endpoint_alias, endpoint="/v1/embeddings", metadata=None + ): + """Record the call and return a stub batch-job id.""" + self.calls.append("create_batch_job") + assert input_file_id == "file-emb" + return {"id": "batch-emb", "status": "validating"} + + async def get_batch_status(self, batch_id, endpoint_alias): + """Return a completed status.""" + self.calls.append("get_batch_status") + return {"status": "completed", "is_complete": True, "progress_percentage": 100} + + async def download_results(self, batch_id, endpoint_alias): + """Return one embedding response body.""" + self.calls.append("download_results") + return { + "success": True, + "responses": [ + { + "custom_id": "e1", + "response": { + "body": {"data": [{"embedding": [0.5, 0.25]}], "usage": {"prompt_tokens": 7}} + }, + } + ], + } + + +def test_pg_embedding_backend_submit_poll_retrieve() -> None: + """Pg embedding backend submit poll retrieve.""" + client = _FakeEmbeddingClient() + backend = PgLlmBatchEmbeddingBackend(client, endpoint_alias="prod_gateway") + job = backend.submit( + [EmbeddingBatchRequest(input_text="embed me", model="embed-x", custom_id="e1")], + metadata={"routing_reason": "bulk"}, + ) + assert job.backend == "pg-llm-batch" + assert job.job_id == "batch-emb" + assert backend.poll(job)["is_complete"] is True + items = backend.retrieve(job) + assert len(items) == 1 + assert items[0].custom_id == "e1" + assert items[0].embedding == [0.5, 0.25] + assert items[0].prompt_tokens == 7 + assert items[0].model == "embed-x" + assert client.calls == ["upload_jsonl", "create_batch_job", "get_batch_status", "download_results"] + + +def test_pg_embedding_backend_incomplete_download_returns_empty() -> None: + """Pg embedding backend incomplete download returns empty.""" + + class _IncompleteClient(_FakeEmbeddingClient): + """Fake client whose result download is explicitly unsuccessful.""" + + async def download_results(self, batch_id, endpoint_alias): + """Report an unsuccessful download.""" + return {"success": False} + + backend = PgLlmBatchEmbeddingBackend(_IncompleteClient()) + job = backend.submit([EmbeddingBatchRequest(input_text="x", model="embed-x", custom_id="e1")]) + assert backend.retrieve(job) == [] + + +class _FakeAssembler: + """Payload assembler stand-in that records the assembled JSONL lines.""" + + def __init__(self) -> None: + """Start without an assembled payload.""" + self.assembled = None + + def assemble(self, lines) -> str: + """Record the lines and return a stub file path.""" + self.assembled = lines + return "file:///tmp/embeddings.jsonl" + + +def test_pg_embedding_backend_uses_payload_assembler_when_provided() -> None: + """Pg embedding backend uses payload assembler when provided.""" + assembler = _FakeAssembler() + backend = PgLlmBatchEmbeddingBackend(_FakeEmbeddingClient(), payload_assembler=assembler) + backend.submit([EmbeddingBatchRequest(input_text="hi", model="embed-x", custom_id="e1")]) + assert assembler.assembled is not None + assert assembler.assembled[0]["custom_id"] == "e1" + + +def test_completions_backend_uses_payload_assembler_when_provided() -> None: + """Completions backend uses payload assembler when provided.""" + assembler = _FakeAssembler() + backend = PgLlmBatchBackend(_FakeEmbeddingClient(), payload_assembler=assembler) + backend.submit( + [BatchRequest(messages=[{"role": "user", "content": "hi"}], custom_id="a", model="gpt-x")] + ) + assert assembler.assembled is not None + assert assembler.assembled[0]["custom_id"] == "a" + + +def test_routing_hints_from_mapping_normalizes_values() -> None: + """Routing hints from mapping normalizes values.""" + hints = RoutingHints.from_mapping( + {"channel": "Batch", "latency_tolerant": True, "priority": "Bulk"} + ) + assert hints.channel == "batch" + assert hints.latency_tolerant is True + assert hints.priority == "bulk" + + +def test_routing_hints_from_mapping_defaults_when_empty() -> None: + """Routing hints from mapping defaults when empty.""" + hints = RoutingHints.from_mapping(None) + assert hints.channel is None + assert hints.priority == "normal" diff --git a/tests/test_cost_ledger_telemetry.py b/tests/test_cost_ledger_telemetry.py new file mode 100644 index 000000000..0ac3cb3a2 --- /dev/null +++ b/tests/test_cost_ledger_telemetry.py @@ -0,0 +1,230 @@ +"""Telemetry, non-blocking, and inline-failure branches of the cost ledger. + +Covers the best-effort telemetry emit + FIFO buffer eviction, the queue-full +drop and flush-timeout paths, the non-blocking worker persistence + emit path, +inline store-failure handling, the in-memory store length, and the SQL +half-open time-window WHERE clauses — all on stdlib fakes, no Postgres. +""" + +from __future__ import annotations + +from pathlib import Path +import queue as queue_mod +import sqlite3 +import sys +import threading + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.cost_ledger import ( # noqa: E402 + AttributionDimensions, + CostLedger, + InMemoryLedgerStore, + InMemoryUsageTelemetrySink, + NonBlockingLedgerStore, + PriceBook, + PriceEntry, + SqlLedgerStore, + UsageTelemetryEvent, + _emit_usage_event, +) +from contextual_orchestrator.kv_config import InMemoryConfigStore # noqa: E402 + +_EXPORT_STATE_KEY = "contextual_orchestrator.usage.export_state" + + +def _ledger(store=None, **kwargs) -> CostLedger: + """Build a priced ledger backed by the given store (in-memory by default).""" + price_book = PriceBook(InMemoryConfigStore()) + price_book.set_price( + PriceEntry("openai", "gpt-x", prompt_price_per_1k=2.0, completion_price_per_1k=4.0) + ) + return CostLedger(price_book, store=store, **kwargs) + + +def _one_record(ledger: CostLedger | None = None): + """Persist and return a single usage record through an inline ledger.""" + ledger = ledger or _ledger() + return ledger.record_usage( + provider="openai", model="gpt-x", prompt_tokens=10, completion_tokens=5 + ) + + +def _export_states(sink: InMemoryUsageTelemetrySink): + """Return the export_state of every event the sink captured.""" + return [event.attributes.get(_EXPORT_STATE_KEY) for event in sink.events()] + + +class _RaisingSink: + """Telemetry sink whose emit always raises, proving emit is best-effort.""" + + def emit_usage(self, event) -> None: + """Always fail, so callers must swallow the error.""" + raise RuntimeError("telemetry backend down") + + +class _AlwaysFullQueue: + """Fake queue that rejects every put and never drains. + + Implements the full interface the ``NonBlockingLedgerStore`` background + worker touches so injecting an instance cannot raise inside the worker. + """ + + def __init__(self) -> None: + """Start with one unfinished task so ``flush`` sees pending work.""" + self.unfinished_tasks = 1 + self._parked = threading.Event() + + def put_nowait(self, item) -> None: + """Reject the record, mimicking a saturated bounded queue.""" + raise queue_mod.Full + + def get(self, *args, **kwargs): + """Park a background worker harmlessly; it never receives an item.""" + self._parked.wait() + + def task_done(self) -> None: + """Do nothing because this fake never dequeues an item.""" + + +def test_emit_usage_event_swallows_sink_errors() -> None: + """A failing telemetry sink must not propagate out of _emit_usage_event.""" + event = UsageTelemetryEvent.from_record(_one_record(), export_state="stored") + assert _emit_usage_event(_RaisingSink(), event) is None + + +def test_in_memory_sink_evicts_oldest_beyond_max_events() -> None: + """At capacity, the sink drops the oldest event and keeps the newest (FIFO).""" + sink = InMemoryUsageTelemetrySink(max_events=1) + record = _one_record() + first = UsageTelemetryEvent.from_record(record, export_state="dropped") + second = UsageTelemetryEvent.from_record(record, export_state="stored") + sink.emit_usage(first) + sink.emit_usage(second) + events = sink.events() + assert len(events) == 1 + assert events[0].attributes[_EXPORT_STATE_KEY] == "stored" + + +def test_in_memory_ledger_store_len_tracks_rows() -> None: + """len(InMemoryLedgerStore) reflects the number of appended rows.""" + store = InMemoryLedgerStore() + assert len(store) == 0 + store.append(_one_record()) + assert len(store) == 1 + + +def test_non_blocking_store_rejects_non_positive_queue_size() -> None: + """A non-positive queue size is rejected at construction.""" + with pytest.raises(ValueError): + NonBlockingLedgerStore(InMemoryLedgerStore(), queue_size=0) + + +def test_non_blocking_store_query_delegates_to_backend() -> None: + """query() delegates straight to the wrapped backend store.""" + store = NonBlockingLedgerStore(InMemoryLedgerStore()) + assert store.query() == [] + + +def test_non_blocking_store_drops_record_when_queue_is_full() -> None: + """A saturated queue drops the record and emits a 'dropped' telemetry event.""" + sink = InMemoryUsageTelemetrySink() + store = NonBlockingLedgerStore(InMemoryLedgerStore(), telemetry_sink=sink) + store._queue = _AlwaysFullQueue() + store.append(_one_record()) + assert "dropped" in _export_states(sink) + + +def test_non_blocking_store_flush_times_out_when_writes_pending() -> None: + """flush() returns False when work stays pending past the deadline.""" + store = NonBlockingLedgerStore(InMemoryLedgerStore()) + store._queue = _AlwaysFullQueue() + assert store.flush(timeout=0.0) is False + + +def test_non_blocking_store_worker_persists_and_emits_stored() -> None: + """An explicitly injected empty backend is preserved and receives worker writes.""" + sink = InMemoryUsageTelemetrySink() + backend = InMemoryLedgerStore() + ledger = _ledger(store=backend, non_blocking_store=True, telemetry_sink=sink) + ledger.record_usage(provider="openai", model="gpt-x", prompt_tokens=10, completion_tokens=5) + assert ledger.flush(timeout=2.0) is True + assert len(backend) == 1 + assert "stored" in _export_states(sink) + + +class _FailingStore: + """Ledger store whose append always raises (inline-failure path).""" + + def append(self, record) -> None: + """Always fail so the ledger records the failure as telemetry only.""" + raise RuntimeError("store down") + + def query(self, start=None, end=None): + """Return no rows; the failing store persists nothing.""" + return [] + + +def test_inline_store_failure_is_recorded_as_telemetry() -> None: + """An inline store failure is swallowed, emitted, and counted, not raised.""" + sink = InMemoryUsageTelemetrySink() + ledger = _ledger(store=_FailingStore(), telemetry_sink=sink) + record = ledger.record_usage( + provider="openai", model="gpt-x", prompt_tokens=1, completion_tokens=1 + ) + assert record is not None + assert "export_error" in _export_states(sink) + assert ledger.telemetry_health()["store_failures"] >= 1 + + +def test_record_usage_accepts_attribution_dimensions_object() -> None: + """record_usage accepts a pre-built AttributionDimensions without remapping.""" + record = _ledger().record_usage( + provider="openai", + model="gpt-x", + prompt_tokens=1, + completion_tokens=1, + attribution=AttributionDimensions(account="acct_one"), + ) + assert record.attribution.account == "acct_one" + + +def test_attribution_from_mapping_uses_provider_alias_for_upstream_api() -> None: + """A loose 'provider' key maps onto the upstream_api dimension.""" + dims = AttributionDimensions.from_mapping({"provider": "prov_alias"}) + assert dims.upstream_api == "prov_alias" + + +def test_telemetry_event_includes_optional_record_dimensions() -> None: + """workflow_run_id and route_mode surface as event attributes when present.""" + record = _ledger().record_usage( + provider="openai", + model="gpt-x", + prompt_tokens=1, + completion_tokens=1, + workflow_run_id="wf_run_1", + route_mode="deep", + ) + event = UsageTelemetryEvent.from_record(record, export_state="stored") + assert event.attributes["contextual_orchestrator.workflow_run_id"] == "wf_run_1" + assert event.attributes["contextual_orchestrator.route_mode"] == "deep" + + +def test_flush_returns_true_when_store_is_synchronous() -> None: + """CostLedger.flush short-circuits to True when the store exposes no flush().""" + assert _ledger().flush() is True + + +def test_sql_ledger_store_query_builds_time_window_clauses() -> None: + """The time-window query includes [start, end) and excludes end itself.""" + conn = sqlite3.connect(":memory:") + try: + store = SqlLedgerStore(conn, paramstyle="qmark") + record = _one_record() + store.append(record) + assert len(store.query(start=0, end=record.created_at + 1)) == 1 + assert store.query(start=0, end=record.created_at) == [] + finally: + conn.close() diff --git a/tests/test_cost_router.py b/tests/test_cost_router.py index 19cf25d10..a9d029f93 100644 --- a/tests/test_cost_router.py +++ b/tests/test_cost_router.py @@ -5,11 +5,14 @@ from pathlib import Path import sys +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ( # noqa: E402 CostLedger, CostRoutingCoordinator, + ConfigBackendUnavailableError, InMemoryUsageTelemetrySink, InMemoryConfigStore, ModelAgent, @@ -19,6 +22,7 @@ TaskOrchestrator, ) from contextual_orchestrator.batch_routing import PgLlmBatchBackend # noqa: E402 +import contextual_orchestrator.cost_router as cost_router_module # noqa: E402 class _FailingLedgerStore: @@ -41,6 +45,66 @@ def _coordinator(ledger=None) -> CostRoutingCoordinator: return CostRoutingCoordinator(orchestrator, config, price_book=price_book, ledger=ledger) +def test_coordinator_uses_configured_durable_store( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A supplied DSN selects the durable config factory, not process memory.""" + orchestrator = TaskOrchestrator( + [ModelAgent("mock_worker", "mock-a", "mock://a", provider_name="mock")] + ) + durable_store = InMemoryConfigStore() + observed: dict[str, object] = {} + + def fake_get_config_store(postgres_dsn: str): + observed["config_dsn"] = postgres_dsn + return durable_store + + def fake_build_token_counter(postgres_dsn: str): + observed["counter_dsn"] = postgres_dsn + return object() + + monkeypatch.setattr(cost_router_module, "get_config_store", fake_get_config_store) + monkeypatch.setattr( + cost_router_module, + "build_token_counter", + fake_build_token_counter, + ) + + coordinator = CostRoutingCoordinator( + orchestrator, + postgres_dsn="postgresql://runtime.example/config", + ) + + assert coordinator.config is durable_store + assert observed == { + "config_dsn": "postgresql://runtime.example/config", + "counter_dsn": "postgresql://runtime.example/config", + } + + +def test_coordinator_propagates_durable_config_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A durable config outage must stop coordinator construction.""" + orchestrator = TaskOrchestrator( + [ModelAgent("mock_worker", "mock-a", "mock://a", provider_name="mock")] + ) + + def fail_config_store(_postgres_dsn: str): + raise ConfigBackendUnavailableError("Postgres config backend is unavailable") + + monkeypatch.setattr(cost_router_module, "get_config_store", fail_config_store) + + with pytest.raises( + ConfigBackendUnavailableError, + match="Postgres config backend is unavailable", + ): + CostRoutingCoordinator( + orchestrator, + postgres_dsn="postgresql://operator:secret@example.invalid/runtime", + ) + + def test_sync_completion_records_usage_and_returns_costs() -> None: coordinator = _coordinator() result = coordinator.complete( diff --git a/tests/test_fuzz_dependency_lock.py b/tests/test_fuzz_dependency_lock.py new file mode 100644 index 000000000..20da24a53 --- /dev/null +++ b/tests/test_fuzz_dependency_lock.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import re +import tomli as tomllib +from dataclasses import dataclass +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +PROJECT_METADATA_PATH = REPOSITORY_ROOT / "pyproject.toml" +FUZZ_LOCK_PATH = REPOSITORY_ROOT / "fuzz" / "requirements-atheris.txt" +MARKER_PATTERN = re.compile( + r"^atheris==(?P\d+\.\d+\.\d+)\s*;\s*" + r"(?Ppython_version|python_full_version)\s*" + r"(?P<|>=)\s*['\"](?P\d+\.\d+)['\"]$" +) +HASH_PATTERN = re.compile(r"--hash=sha256:(?P[0-9a-f]{64})") + + +@dataclass(frozen=True) +class InterpreterRequirement: + """One interpreter-gated Atheris release parsed from project or lock data.""" + + release: str + field: str + operator: str + boundary: tuple[int, int] + hashes: frozenset[str] = frozenset() + + def matches(self, python_version: tuple[int, int]) -> bool: + """Return whether the requirement applies to one Python major/minor pair.""" + + if self.operator == "<": + return python_version < self.boundary + if self.operator == ">=": + return python_version >= self.boundary + raise AssertionError(f"Unsupported marker operator: {self.operator}") + + +def _version_pair(value: str) -> tuple[int, int]: + """Parse a dotted major/minor version into a comparable integer pair.""" + + major, minor = value.split(".", 1) + return int(major), int(minor) + + +def _parse_requirement(requirement: str) -> InterpreterRequirement: + """Parse the deliberately narrow Atheris marker grammar used by this lock.""" + + match = MARKER_PATTERN.fullmatch(requirement.strip()) + assert match is not None, f"Unexpected Atheris requirement: {requirement!r}" + return InterpreterRequirement( + release=match.group("release"), + field=match.group("field"), + operator=match.group("operator"), + boundary=_version_pair(match.group("boundary")), + ) + + +def _project_requirements() -> tuple[InterpreterRequirement, ...]: + """Load all Atheris requirements declared by the project's fuzz extra.""" + + metadata = tomllib.loads(PROJECT_METADATA_PATH.read_text(encoding="utf-8")) + fuzz_extra = metadata["project"]["optional-dependencies"]["fuzz"] + return tuple( + _parse_requirement(requirement) + for requirement in fuzz_extra + if requirement.startswith("atheris==") + ) + + +def _lock_requirements() -> tuple[InterpreterRequirement, ...]: + """Load interpreter markers and SHA-256 evidence from the universal lock.""" + + lines = FUZZ_LOCK_PATH.read_text(encoding="utf-8").splitlines() + requirements: list[InterpreterRequirement] = [] + line_index = 0 + while line_index < len(lines): + line = lines[line_index] + if not line.startswith("atheris=="): + line_index += 1 + continue + + assert line.rstrip().endswith("\\"), ( + f"Atheris lock header must continue to hashes: {line!r}" + ) + parsed = _parse_requirement(line.rstrip()[:-1].rstrip()) + line_index += 1 + hashes: set[str] = set() + while line_index < len(lines) and lines[line_index].startswith((" ", "\t")): + hash_match = HASH_PATTERN.search(lines[line_index]) + if hash_match is not None: + hashes.add(hash_match.group("digest")) + line_index += 1 + requirements.append( + InterpreterRequirement( + release=parsed.release, + field=parsed.field, + operator=parsed.operator, + boundary=parsed.boundary, + hashes=frozenset(hashes), + ) + ) + return tuple(requirements) + + +def _selected_release( + requirements: tuple[InterpreterRequirement, ...], + python_version: tuple[int, int], +) -> str: + """Require exactly one applicable release for a representative interpreter.""" + + selected = [ + requirement.release + for requirement in requirements + if requirement.matches(python_version) + ] + assert len(selected) == 1, ( + f"Expected exactly one Atheris release for Python {python_version}, " + f"selected {selected}" + ) + return selected[0] + + +def test_fuzz_extra_selects_one_published_release_per_supported_interpreter() -> None: + """Project metadata must partition supported interpreters without gaps or overlap.""" + + requirements = _project_requirements() + assert { + (entry.release, entry.field, entry.operator, entry.boundary) + for entry in requirements + } == { + ("3.0.0", "python_version", "<", (3, 13)), + ("3.1.0", "python_version", ">=", (3, 13)), + } + assert _selected_release(requirements, (3, 11)) == "3.0.0" + assert _selected_release(requirements, (3, 13)) == "3.1.0" + assert _selected_release(requirements, (3, 14)) == "3.1.0" + + +def test_universal_lock_matches_markers_and_published_hashes() -> None: + """The universal lock must mirror project markers and hash every selected wheel.""" + + requirements = _lock_requirements() + assert { + (entry.release, entry.field, entry.operator, entry.boundary) + for entry in requirements + } == { + ("3.0.0", "python_full_version", "<", (3, 13)), + ("3.1.0", "python_full_version", ">=", (3, 13)), + } + assert _selected_release(requirements, (3, 11)) == "3.0.0" + assert _selected_release(requirements, (3, 13)) == "3.1.0" + assert _selected_release(requirements, (3, 14)) == "3.1.0" + + hashes_by_release = {entry.release: entry.hashes for entry in requirements} + assert hashes_by_release["3.0.0"] == { + "1f0929c7bc3040f3fe4102e557718734190cf2d7718bbb8e3ce6d3eb56ef5bb3", + "510e502c57b6dc615fb174066407af620d4c7f73cf08a782c86e7761bf12c4eb", + "8a5c8a781467c187da40fd29139784193e2647058831f837f675d0bb8cbd8746", + "a402cdca8a650d1371050b1f9552eb4cdc488d2db64950d603c4560318365eac", + } + assert hashes_by_release["3.1.0"] == { + "315a0b5c819852b1ffe1ca72efc389c7724881f2c33e4aacb8c6bcec49bd5011", + "ec5e11f21a4c197fe91f7aea2b2de88e623c73a21fc07b105ac6329a1588457b", + "f8a9f51ce8369026e8eb7b7174835e8c4c85a1a6db5d9add36c15100779d2a39", + } diff --git a/tests/test_kv_config.py b/tests/test_kv_config.py new file mode 100644 index 000000000..aea52385d --- /dev/null +++ b/tests/test_kv_config.py @@ -0,0 +1,281 @@ +"""KV config-store seam: in-memory store, Postgres adapter, and factory. + +Runs entirely on the dependency-free in-memory backend plus lightweight fakes +standing in for the ``pg_llm_batch`` Postgres config/secret stores — no +Postgres or ``pg_llm_batch`` install required. +""" + +from __future__ import annotations + +from pathlib import Path +import sys +import traceback + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.kv_config import ( # noqa: E402 + ConfigBackendUnavailableError, + InMemoryConfigStore, + PostgresConfigStoreAdapter, + get_config_store, +) + + +def test_in_memory_store_seeds_nested_entries() -> None: + """A seed mapping is loaded into the nested category/key store.""" + store = InMemoryConfigStore(seed={"price_table": {"gpt_example": 1.0}}) + assert store.get("price_table", "gpt_example") == 1.0 + + +def test_in_memory_get_returns_default_when_unset() -> None: + """get() returns the supplied default for an unset key.""" + store = InMemoryConfigStore() + assert store.get("routing_policy", "missing_key", "fallback_value") == "fallback_value" + + +def test_in_memory_get_category_returns_defensive_copy() -> None: + """get_category returns a copy that cannot mutate the backing store.""" + store = InMemoryConfigStore(seed={"routing_policy": {"sync_threshold": 5}}) + category = store.get_category("routing_policy") + assert category == {"sync_threshold": 5} + category["sync_threshold"] = 99 + assert store.get("routing_policy", "sync_threshold") == 5 + + +def test_in_memory_get_category_missing_is_empty() -> None: + """get_category returns an empty dict for an unknown category.""" + assert InMemoryConfigStore().get_category("absent_category") == {} + + +def test_in_memory_show_config_yields_sorted_entries() -> None: + """show_config yields every (category, key, value) in sorted order.""" + store = InMemoryConfigStore() + store.set("beta_group", "second_key", 2) + store.set("alpha_group", "first_key", 1) + assert list(store.show_config()) == [ + ("alpha_group", "first_key", 1), + ("beta_group", "second_key", 2), + ] + + +def test_in_memory_secret_roundtrip_and_not_shown_in_config() -> None: + """Secrets round-trip through set/get and never appear in show_config.""" + store = InMemoryConfigStore() + store.set_secret("openai_api_key", "sk-live") + assert store.get_secret("openai_api_key") == "sk-live" + assert list(store.show_config()) == [] + + +def test_in_memory_get_secret_default_when_absent() -> None: + """get_secret returns the supplied default for an unknown secret.""" + assert InMemoryConfigStore().get_secret("absent_secret", "default_value") == "default_value" + + +def test_in_memory_require_secret_returns_stored_value() -> None: + """require_secret returns the stored value when the secret exists.""" + store = InMemoryConfigStore() + store.set_secret("api_token", "value_one") + assert store.require_secret("api_token") == "value_one" + + +def test_in_memory_require_secret_raises_when_missing() -> None: + """require_secret raises KeyError for an unconfigured secret.""" + with pytest.raises(KeyError): + InMemoryConfigStore().require_secret("absent_secret") + + +class _FakeConfigStore: + """Minimal stand-in for ``pg_llm_batch.PostgresConfigStore``.""" + + def __init__(self) -> None: + """Start with an empty key/value map.""" + self._pairs: dict = {} + + def get(self, category: str, key: str, default=None): + """Return the value under ``(category, key)`` or ``default``.""" + return self._pairs.get((category, key), default) + + def set(self, category: str, key: str, value) -> None: + """Store ``value`` under ``(category, key)``.""" + self._pairs[(category, key)] = value + + +class _FakeSecretStore: + """Minimal stand-in for ``pg_llm_batch.SecretStore``.""" + + def __init__(self, known=None) -> None: + """Seed the fake with a mapping of known secrets.""" + self._known = dict(known or {}) + + def require_secret(self, secret_name: str) -> str: + """Return a known secret or raise, mirroring the real store.""" + if secret_name not in self._known: + raise KeyError(secret_name) + return self._known[secret_name] + + +def test_postgres_adapter_delegates_get_and_set() -> None: + """The adapter forwards get/set to the backing config store.""" + adapter = PostgresConfigStoreAdapter(_FakeConfigStore()) + adapter.set("price_table", "gpt_example", 2.5) + assert adapter.get("price_table", "gpt_example") == 2.5 + assert adapter.get("price_table", "absent_key", "default_value") == "default_value" + + +def test_postgres_adapter_get_secret_without_store_returns_default() -> None: + """With no secret store, get_secret returns the default.""" + adapter = PostgresConfigStoreAdapter(_FakeConfigStore(), secret_store=None) + assert adapter.get_secret("openai_api_key", "fallback_value") == "fallback_value" + + +def test_postgres_adapter_get_secret_returns_backing_value() -> None: + """get_secret returns the value from the backing secret store.""" + adapter = PostgresConfigStoreAdapter(_FakeConfigStore(), _FakeSecretStore({"api_token": "secret_v"})) + assert adapter.get_secret("api_token") == "secret_v" + + +def test_postgres_adapter_get_secret_swallows_backend_error() -> None: + """get_secret returns the default when the backing store raises.""" + adapter = PostgresConfigStoreAdapter(_FakeConfigStore(), _FakeSecretStore({})) + assert adapter.get_secret("absent_secret", "default_on_error") == "default_on_error" + + +def test_postgres_adapter_require_secret_without_store_raises() -> None: + """require_secret raises KeyError when no secret store is configured.""" + adapter = PostgresConfigStoreAdapter(_FakeConfigStore(), secret_store=None) + with pytest.raises(KeyError): + adapter.require_secret("api_token") + + +def test_postgres_adapter_require_secret_delegates() -> None: + """require_secret delegates to the backing secret store.""" + adapter = PostgresConfigStoreAdapter(_FakeConfigStore(), _FakeSecretStore({"api_token": "value_one"})) + assert adapter.require_secret("api_token") == "value_one" + + +def test_get_config_store_without_dsn_is_in_memory() -> None: + """With no DSN the factory returns a seeded in-memory config store.""" + store = get_config_store(seed={"price_table": {"gpt_example": 1.0}}) + assert isinstance(store, InMemoryConfigStore) + assert store.get("price_table", "gpt_example") == 1.0 + + +def test_get_config_store_with_dsn_fails_closed_when_dependency_is_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A configured durable backend must not silently become process-local.""" + monkeypatch.setitem(sys.modules, "pg_llm_batch", None) + + with pytest.raises( + ConfigBackendUnavailableError, + match="Postgres config backend is unavailable", + ): + get_config_store("postgresql://operator:secret@example.invalid/runtime") + + +def test_get_config_store_with_dsn_fails_closed_without_disclosing_dsn( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Backend initialization failures stay fatal without echoing credentials.""" + class FailingConfigStore: + """Stand-in whose constructor reproduces a backend outage.""" + + def __init__(self, _postgres_dsn: str) -> None: + raise OSError("could not reach postgresql://operator:secret@example.invalid/runtime") + + class UnusedSecretStore: + """Constructor placeholder that must not be reached after config failure.""" + + def __init__(self, _postgres_dsn: str, *, fernet_key=None) -> None: + raise AssertionError("secret store construction must not be reached") + + fake_module = type(sys)("pg_llm_batch") + fake_module.PostgresConfigStore = FailingConfigStore + fake_module.SecretStore = UnusedSecretStore + monkeypatch.setitem(sys.modules, "pg_llm_batch", fake_module) + + configured_dsn = "postgresql://operator:secret@example.invalid/runtime" + with pytest.raises(ConfigBackendUnavailableError) as caught: + get_config_store(configured_dsn) + + assert str(caught.value) == "Postgres config backend is unavailable" + assert "operator" not in str(caught.value) + assert "secret" not in str(caught.value) + rendered_traceback = "".join( + traceback.format_exception(caught.type, caught.value, caught.tb) + ) + assert "operator" not in rendered_traceback + assert "secret" not in rendered_traceback + + +def test_get_config_store_with_dsn_builds_and_seeds_postgres_adapter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A healthy configured backend receives bootstrap inputs and the seed.""" + constructed: dict[str, object] = {} + + class FakePostgresConfigStore(_FakeConfigStore): + """Record the DSN while providing the normal config-store surface.""" + + def __init__(self, postgres_dsn: str) -> None: + super().__init__() + constructed["config_dsn"] = postgres_dsn + + class FakePostgresSecretStore(_FakeSecretStore): + """Record the secret-store bootstrap inputs.""" + + def __init__(self, postgres_dsn: str, *, fernet_key=None) -> None: + super().__init__({"provider_key": "stored_secret"}) + constructed["secret_dsn"] = postgres_dsn + constructed["fernet_key"] = fernet_key + + fake_module = type(sys)("pg_llm_batch") + fake_module.PostgresConfigStore = FakePostgresConfigStore + fake_module.SecretStore = FakePostgresSecretStore + monkeypatch.setitem(sys.modules, "pg_llm_batch", fake_module) + + store = get_config_store( + "postgresql://runtime.example/config", + fernet_key="bootstrap_key", + seed={"routing_policy": {"sync_threshold": 4}}, + ) + + assert isinstance(store, PostgresConfigStoreAdapter) + assert store.get("routing_policy", "sync_threshold") == 4 + assert store.require_secret("provider_key") == "stored_secret" + assert constructed == { + "config_dsn": "postgresql://runtime.example/config", + "secret_dsn": "postgresql://runtime.example/config", + "fernet_key": "bootstrap_key", + } + + +def test_get_config_store_with_dsn_accepts_an_empty_seed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A healthy durable backend does not require bootstrap config entries.""" + fake_module = type(sys)("pg_llm_batch") + fake_module.PostgresConfigStore = lambda _postgres_dsn: _FakeConfigStore() + fake_module.SecretStore = ( + lambda _postgres_dsn, *, fernet_key=None: _FakeSecretStore({}) + ) + monkeypatch.setitem(sys.modules, "pg_llm_batch", fake_module) + + store = get_config_store("postgresql://runtime.example/config") + + assert isinstance(store, PostgresConfigStoreAdapter) + assert store.get("routing_policy", "missing", "default") == "default" + + +def test_kv_docs_require_fail_closed_durable_backend() -> None: + """Operator guidance must forbid an implicit durable-to-memory downgrade.""" + repository_root = Path(__file__).resolve().parents[1] + guidance = (repository_root / "docs" / "kv-credentials.md").read_text( + encoding="utf-8" + ) + + assert "never silently falls back" in guidance + assert "Postgres config backend is unavailable" in guidance + assert "intentionally select `memory`" in guidance diff --git a/tests/test_kv_credentials.py b/tests/test_kv_credentials.py index 055c591b1..184f4637c 100644 --- a/tests/test_kv_credentials.py +++ b/tests/test_kv_credentials.py @@ -13,10 +13,12 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from contextual_orchestrator import ModelAgent # noqa: E402 +from contextual_orchestrator import ModelAgent, credentials # noqa: E402 from contextual_orchestrator.credentials import ( # noqa: E402 InMemoryCredentialBackend, NotConfigured, + PostgresCredentialBackend, + _select_backend, get_credential, register_credential, set_backend, @@ -35,35 +37,37 @@ def _fresh_backend(): def test_get_credential_returns_none_when_absent() -> None: + """Unregistered credentials are absent from the in-memory backend.""" assert get_credential("OPENAI_API_KEY") is None def test_register_then_get_roundtrips_via_kv() -> None: + """Registered credentials round-trip through the active KV backend.""" register_credential("OPENAI_API_KEY", "sk-live-123") assert get_credential("OPENAI_API_KEY") == "sk-live-123" def test_register_credential_overwrites() -> None: + """A later registration replaces the prior credential value.""" register_credential("OPENAI_API_KEY", "sk-old") register_credential("OPENAI_API_KEY", "sk-new") assert get_credential("OPENAI_API_KEY") == "sk-new" def test_credential_key_defaults_to_openai() -> None: + """Remote agents default to the reviewed OpenAI credential key name.""" agent = ModelAgent("remote_agent", "gpt-example", "https://api.openai.com/v1") assert agent.credential_key == "OPENAI_API_KEY" assert agent.credential_name == "OPENAI_API_KEY" def test_legacy_api_key_env_is_treated_as_credential_name_not_env() -> None: - # A legacy api_key_env value maps to the credential NAME; it is never read - # from the process environment. + """Legacy api_key_env names never authorize a process-environment fallback.""" os.environ.pop("LEGACY_PROVIDER_KEY", None) os.environ["LEGACY_PROVIDER_KEY"] = "sk-from-env-should-be-ignored" try: agent = ModelAgent("legacy_agent", "gpt-example", "https://api.openai.com/v1", "LEGACY_PROVIDER_KEY") assert agent.credential_name == "LEGACY_PROVIDER_KEY" - # Not registered in the KV -> unresolvable, despite the env var existing. assert get_credential(agent.credential_name) is None register_credential("LEGACY_PROVIDER_KEY", "sk-from-kv") assert get_credential(agent.credential_name) == "sk-from-kv" @@ -72,6 +76,7 @@ def test_legacy_api_key_env_is_treated_as_credential_name_not_env() -> None: def test_explicit_credential_key_resolves() -> None: + """An explicit provider credential key resolves only through the KV seam.""" agent = ModelAgent( "vendor_agent", "gpt-example", "https://api.openai.com/v1", credential_key="VENDOR_API_KEY" ) @@ -81,8 +86,7 @@ def test_explicit_credential_key_resolves() -> None: def test_non_mock_agent_without_credential_raises_not_env_fallback() -> None: - # Even with a matching env var set, an unresolved KV credential must raise - # NotConfigured rather than silently reading os.getenv. + """Remote validation fails closed when KV has no credential, even if env does.""" os.environ["OPENAI_API_KEY"] = "sk-env-must-not-be-used" try: client = ModelClient() @@ -95,13 +99,14 @@ def test_non_mock_agent_without_credential_raises_not_env_fallback() -> None: def test_mock_agent_stays_keyless() -> None: - # Mock agents early-return before any credential logic; no KV required. + """Mock agents require no remote credential material.""" client = ModelClient() agent = ModelAgent("general_agent", "mock-generalist", "mock://local") assert client.chat(agent, [{"role": "user", "content": "hi"}]) def test_unknown_backend_selector_raises(monkeypatch) -> None: + """Unknown credential backends fail closed rather than silently falling back.""" from contextual_orchestrator import credentials set_backend(None) @@ -109,3 +114,72 @@ def test_unknown_backend_selector_raises(monkeypatch) -> None: with pytest.raises(NotConfigured): credentials.get_backend() set_backend(None) + + +def test_postgres_backend_requires_bootstrap_dsn() -> None: + """A missing bootstrap DSN fails closed with NotConfigured.""" + with pytest.raises(NotConfigured): + PostgresCredentialBackend("", "boot-passphrase") + + +def test_postgres_backend_requires_bootstrap_passphrase() -> None: + """A missing bootstrap passphrase fails closed with NotConfigured.""" + with pytest.raises(NotConfigured): + PostgresCredentialBackend("postgresql://host/db", "") + + +def test_postgres_backend_stores_bootstrap_transport() -> None: + """A valid dsn+passphrase is stored as bootstrap transport, schema not yet ensured.""" + backend = PostgresCredentialBackend("postgresql://host/db", "boot-passphrase") + assert backend._dsn == "postgresql://host/db" + assert backend._passphrase == "boot-passphrase" # noqa: S105 - test-only fixture + assert backend._ensured is False + + +def test_postgres_backend_from_env_reads_bootstrap_vars(monkeypatch) -> None: + """from_env reads both the DSN and passphrase bootstrap env vars.""" + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_KV_DSN", "postgresql://host/db") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE", "boot-passphrase") + backend = PostgresCredentialBackend.from_env() + assert isinstance(backend, PostgresCredentialBackend) + assert backend._dsn == "postgresql://host/db" + assert backend._passphrase == "boot-passphrase" # noqa: S105 - test-only fixture + + +def test_select_backend_memory_default_is_in_memory(monkeypatch) -> None: + """The default (unset) backend selector yields the in-memory backend.""" + monkeypatch.delenv("CONTEXTUAL_ORCHESTRATOR_KV_BACKEND", raising=False) + assert isinstance(_select_backend(), InMemoryCredentialBackend) + + +def test_select_backend_postgres_builds_from_env(monkeypatch) -> None: + """Selecting 'postgres' builds the Postgres backend from bootstrap env vars.""" + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_KV_BACKEND", "postgres") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_KV_DSN", "postgresql://host/db") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE", "boot-passphrase") + backend = _select_backend() + assert isinstance(backend, PostgresCredentialBackend) + + +def test_get_backend_reuses_backend_initialized_while_waiting_for_lock(monkeypatch) -> None: + """A concurrent initializer wins without a second backend construction.""" + set_backend(None) + sentinel = InMemoryCredentialBackend() + selector_calls: list[bool] = [] + + class _InitializingLock: + def __enter__(self): + credentials._backend = sentinel + + def __exit__(self, exc_type, exc, traceback): + return False + + monkeypatch.setattr(credentials, "_backend_lock", _InitializingLock()) + monkeypatch.setattr( + credentials, + "_select_backend", + lambda: selector_calls.append(True) or InMemoryCredentialBackend(), + ) + + assert credentials.get_backend() is sentinel + assert selector_calls == [] diff --git a/tests/test_main_cli_coverage.py b/tests/test_main_cli_coverage.py new file mode 100644 index 000000000..81f880202 --- /dev/null +++ b/tests/test_main_cli_coverage.py @@ -0,0 +1,266 @@ +"""Behavioural coverage for the package command-line entrypoint.""" + +from __future__ import annotations + +import io +import json +from pathlib import Path +import runpy +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import contextual_orchestrator.__main__ as cli # noqa: E402 + + +class _FakeOrchestrator: + """Minimal CLI-facing orchestrator that records requested operations.""" + + instances: list["_FakeOrchestrator"] = [] + + def __init__(self, agents, **kwargs) -> None: + self.agents = agents + self.kwargs = kwargs + self.complete_calls: list[tuple[list[dict], str]] = [] + self.eval_calls: list[tuple[list[str], str]] = [] + type(self).instances.append(self) + + def complete(self, messages: list[dict], mode: str = "auto") -> dict: + self.complete_calls.append((messages, mode)) + return {"answer": "cli-answer", "mode": mode} + + def compare_to_baseline(self, prompts: list[str], mode: str = "auto") -> dict: + self.eval_calls.append((prompts, mode)) + return {"prompts": prompts, "mode": mode, "winner": "orchestrator"} + + +def _patch_runtime(monkeypatch: pytest.MonkeyPatch) -> None: + _FakeOrchestrator.instances.clear() + monkeypatch.setattr(cli, "ModelClient", lambda **kwargs: {"client_kwargs": kwargs}) + monkeypatch.setattr(cli, "load_agents", lambda path: [{"loaded_from": path}]) + monkeypatch.setattr(cli, "TaskOrchestrator", _FakeOrchestrator) + + +def test_register_credential_reads_stdin_and_reports_only_name( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Bootstrap stdin transport stores the value without echoing the secret.""" + captured: list[tuple[str, str]] = [] + monkeypatch.setattr(cli, "register_credential", lambda name, value: captured.append((name, value))) + monkeypatch.setattr(sys, "stdin", io.StringIO("super-secret\n")) + monkeypatch.setattr(sys, "argv", ["contextual-orchestrator", "register-credential", "--name", "NVIDIA_NIM_API_KEY", "--value-stdin"]) + + cli.main() + + assert captured == [("NVIDIA_NIM_API_KEY", "super-secret")] + output = json.loads(capsys.readouterr().out) + assert output == {"registered": "NVIDIA_NIM_API_KEY", "backend": "kv"} + assert "super-secret" not in repr(output) + + +def test_register_credential_reads_named_bootstrap_environment( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Explicit environment bootstrap transport resolves only the named variable.""" + captured: list[tuple[str, str]] = [] + monkeypatch.setattr(cli, "register_credential", lambda name, value: captured.append((name, value))) + monkeypatch.setenv("BOOTSTRAP_SECRET", "from-env-secret") + monkeypatch.setattr( + sys, + "argv", + ["contextual-orchestrator", "register-credential", "--name", "provider_key", "--from-env", "BOOTSTRAP_SECRET"], + ) + + cli.main() + + assert captured == [("provider_key", "from-env-secret")] + assert json.loads(capsys.readouterr().out)["registered"] == "provider_key" + + +def test_register_credential_rejects_missing_environment_transport(monkeypatch: pytest.MonkeyPatch) -> None: + """Bootstrap fails closed when the requested transport variable is absent.""" + monkeypatch.delenv("MISSING_BOOTSTRAP_SECRET", raising=False) + monkeypatch.setattr( + sys, + "argv", + ["contextual-orchestrator", "register-credential", "--name", "provider_key", "--from-env", "MISSING_BOOTSTRAP_SECRET"], + ) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 2 + + +def test_register_credential_rejects_empty_value(monkeypatch: pytest.MonkeyPatch) -> None: + """An empty stdin secret is rejected before it reaches the credential backend.""" + monkeypatch.setattr(sys, "stdin", io.StringIO(" \n")) + monkeypatch.setattr(sys, "argv", ["contextual-orchestrator", "register-credential", "--name", "provider_key"]) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 2 + + +def test_prompt_mode_wires_runtime_options_and_prints_completion( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """CLI prompt mode carries routing, storage, TLS, budget, and cache settings.""" + _patch_runtime(monkeypatch) + monkeypatch.setattr( + sys, + "argv", + [ + "contextual-orchestrator", + "explain the result", + "--agents", + "agents.json", + "--state-db", + "runs.sqlite", + "--agents-db", + "agents.sqlite", + "--mode", + "route", + "--provider-ca-bundle", + "corp-ca.pem", + "--insecure-skip-tls-verify", + "--budget-max-output-tokens", + "100", + "--budget-max-cost-usd", + "2.5", + "--cache-ttl", + "4.0", + ], + ) + + cli.main() + + instance = _FakeOrchestrator.instances[-1] + assert instance.agents == [{"loaded_from": "agents.json"}] + assert instance.kwargs["state_db"] == "runs.sqlite" + assert instance.kwargs["agents_db"] == "agents.sqlite" + assert instance.kwargs["budget_max_output_tokens"] == 100 + assert instance.kwargs["budget_max_cost_usd"] == 2.5 + assert instance.kwargs["cache_ttl"] == 4.0 + assert instance.kwargs["client"] == {"client_kwargs": {"ca_bundle": "corp-ca.pem", "verify_tls": False}} + assert instance.complete_calls == [([{"role": "user", "content": "explain the result"}], "route")] + assert json.loads(capsys.readouterr().out) == {"answer": "cli-answer", "mode": "route"} + + +def test_eval_mode_prints_comparable_baseline_report( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Evaluation mode routes all supplied prompts through baseline comparison.""" + _patch_runtime(monkeypatch) + monkeypatch.setattr( + sys, + "argv", + ["contextual-orchestrator", "--mode", "conduct", "--eval", "prompt one", "prompt two"], + ) + + cli.main() + + instance = _FakeOrchestrator.instances[-1] + assert instance.eval_calls == [(["prompt one", "prompt two"], "conduct")] + report = json.loads(capsys.readouterr().out) + assert report["winner"] == "orchestrator" + assert report["prompts"] == ["prompt one", "prompt two"] + + +def test_serve_requires_an_authentication_token(monkeypatch: pytest.MonkeyPatch) -> None: + """Server mode fails closed when no shared or split token is configured.""" + _patch_runtime(monkeypatch) + monkeypatch.delenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", raising=False) + monkeypatch.delenv("CONTEXTUAL_ORCHESTRATOR_ADMIN_TOKEN", raising=False) + monkeypatch.delenv("CONTEXTUAL_ORCHESTRATOR_INFERENCE_TOKEN", raising=False) + monkeypatch.setattr(sys, "argv", ["contextual-orchestrator", "--serve"]) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 2 + + +def test_serve_rejects_incomplete_split_token_mode(monkeypatch: pytest.MonkeyPatch) -> None: + """Split-scope authentication requires both administrator and inference tokens.""" + _patch_runtime(monkeypatch) + monkeypatch.setattr(sys, "argv", ["contextual-orchestrator", "--serve", "--admin-token", "admin-only"]) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 2 + + +def test_serve_wires_security_and_clearfolio_options(monkeypatch: pytest.MonkeyPatch) -> None: + """Authenticated server mode forwards its explicit network and security posture.""" + _patch_runtime(monkeypatch) + captured: dict = {} + + def fake_serve(orchestrator, **kwargs) -> None: + captured["orchestrator"] = orchestrator + captured.update(kwargs) + + monkeypatch.setattr(cli, "serve", fake_serve) + monkeypatch.setattr( + sys, + "argv", + [ + "contextual-orchestrator", + "--serve", + "--host", + "0.0.0.0", + "--port", + "9010", + "--admin-token", + "admin-token", + "--inference-token", + "inference-token", + "--allow-public-bind", + "--expose-trace-by-default", + "--clearfolio-url", + "https://clearfolio.example", + "--insecure-disable-auth", + ], + ) + + cli.main() + + security = captured["security"] + assert captured["host"] == "0.0.0.0" + assert captured["port"] == 9010 + assert captured["clearfolio_url"] == "https://clearfolio.example" + assert security.auth_token == "" + assert security.admin_token == "admin-token" + assert security.inference_token == "inference-token" + assert security.allow_public_bind is True + assert security.expose_trace_by_default is True + + +def test_serve_accepts_legacy_shared_token(monkeypatch: pytest.MonkeyPatch) -> None: + """The shared-token compatibility path remains accepted for authenticated serving.""" + _patch_runtime(monkeypatch) + calls: list[dict] = [] + monkeypatch.setattr(cli, "serve", lambda orchestrator, **kwargs: calls.append(kwargs)) + monkeypatch.setattr(sys, "argv", ["contextual-orchestrator", "--serve", "--auth-token", "shared-token"]) + + cli.main() + + assert calls[0]["security"].auth_token == "shared-token" + + +def test_missing_prompt_is_a_usage_error(monkeypatch: pytest.MonkeyPatch) -> None: + """Non-server execution requires a prompt when evaluation mode is absent.""" + _patch_runtime(monkeypatch) + monkeypatch.setattr(sys, "argv", ["contextual-orchestrator"]) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 2 + + +def test_module_execution_runs_real_mock_cli_path(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """Executing the package module as ``__main__`` reaches the documented mock default.""" + monkeypatch.setattr(sys, "argv", ["contextual-orchestrator", "hello from module execution", "--mode", "route"]) + runpy.run_module("contextual_orchestrator.__main__", run_name="__main__") + output = json.loads(capsys.readouterr().out) + assert isinstance(output.get("answer"), str) + assert output["answer"] diff --git a/tests/test_orchestrator_coverage.py b/tests/test_orchestrator_coverage.py new file mode 100644 index 000000000..73a203995 --- /dev/null +++ b/tests/test_orchestrator_coverage.py @@ -0,0 +1,457 @@ +"""Behavioural coverage for orchestration and commercial-readiness branches. + +The cases in this module exercise public and internal decision paths that are +not reached by the feature suites: provider-client TLS validation, non-mock +chat delegation, Batch parsing, Responses input coercion, generated planning, +model-judge fallback, lifecycle and budget handling, criterion helpers, cache +normalization, and blocked commercial-readiness classifications. Every case +asserts product behaviour rather than executing lines only. +""" + +from __future__ import annotations + +from dataclasses import replace +import inspect +import json +from pathlib import Path +import sys +import tempfile + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.admin import ADMIN_TRANSLATIONS # noqa: E402 +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + register_credential, + set_backend, +) +from contextual_orchestrator.orchestrator import ( # noqa: E402 + BudgetExceededError, + ModelClient, + _classify_commercial_status, + _coerce_input_text, + _freeze_report_cache_value, + _recommend_config, +) + + +def test_build_ssl_context_rejects_unloadable_ca_bundle() -> None: + """A present but invalid custom CA bundle fails with a bounded ValueError.""" + with tempfile.NamedTemporaryFile("w", suffix=".pem", delete=False) as handle: + handle.write("this is not a certificate\n") + bundle_path = handle.name + try: + with pytest.raises(ValueError) as exc: + ModelClient._build_ssl_context(bundle_path, verify_tls=True) + assert "could not be loaded" in str(exc.value) + finally: + Path(bundle_path).unlink() + + +def test_chat_delegates_non_mock_agent_to_send_with_retry() -> None: + """Configured non-mock chat validates then delegates to the retrying sender.""" + set_backend(InMemoryCredentialBackend()) + register_credential("OPENAI_API_KEY", "sk-live") + try: + client = ModelClient() + client._validate_provider = lambda agent: None # type: ignore[assignment] + client._send_with_retry = lambda agent, payload: "provider replied" # type: ignore[assignment] + agent = ModelAgent("remote_agent", "gpt-x", "https://api.example-provider.com/v1") + assert client.chat(agent, [{"role": "user", "content": "hi"}]) == "provider replied" + finally: + set_backend(None) + + +def test_batch_run_parses_results_and_ignores_blank_lines() -> None: + """Batch result parsing preserves rows and ignores blank JSONL separators.""" + + class _FakeBatchClient(ModelClient): + def _batch_upload(self, agent, payload): # type: ignore[override] + return "file-in" + + def _batch_json(self, agent, method, path, payload=None): # type: ignore[override] + if method == "POST" and path == "/batches": + return {"id": "batch-1"} + if path.startswith("/batches/"): + return {"status": "completed", "output_file_id": "file-out"} + raise AssertionError(f"unexpected batch call: {method} {path}") + + def _batch_raw(self, agent, path): # type: ignore[override] + rows = [ + json.dumps( + { + "custom_id": "task_0", + "response": { + "body": { + "choices": [{"message": {"content": "A"}}], + "usage": {"completion_tokens": 3}, + } + }, + } + ), + "", + json.dumps( + { + "custom_id": "task_1", + "response": {"body": {"choices": [{"message": {"content": "B"}}]}}, + } + ), + ] + return "\n".join(rows).encode("utf-8") + + client = _FakeBatchClient() + client._sleep = lambda _seconds: None + agent = ModelAgent("remote_agent", "gpt-x", "https://api.example-provider.com/v1") + results = client._batch_run( + agent, + {"task_0": [{"role": "user", "content": "a"}], "task_1": [{"role": "user", "content": "b"}]}, + temperature=0.2, + poll_interval=0.0, + poll_timeout=5.0, + ) + assert results["task_0"]["content"] == "A" + assert results["task_0"]["usage"] == {"completion_tokens": 3} + assert results["task_1"]["content"] == "B" + assert results["task_1"]["usage"] is None + + +def test_coerce_input_text_flattens_strings_dicts_and_content_lists() -> None: + """Responses input coercion walks only supported text-bearing shapes.""" + value = [ + "plain string", + 42, + {"content": "dict content"}, + {"content": [{"text": "chunk text"}, {"no_text": 1}]}, + {"other": "ignored"}, + ] + assert _coerce_input_text(value) == "plain string dict content chunk text" + assert _coerce_input_text("already a string") == "already a string" + assert _coerce_input_text(None) == "" + + +_GENERATED_PLAN = { + "steps": [ + {"id": 0, "role": "worker", "agent_id": "general_agent", "subtask": "Draft the answer.", "access": []}, + {"id": 1, "role": "worker", "agent_id": "general_agent", "subtask": "Draft an alternative.", "access": []}, + {"id": 2, "role": "verifier", "agent_id": "general_agent", "subtask": "Check both drafts.", "access": [0, 1]}, + {"id": 3, "role": "synthesizer", "agent_id": "general_agent", "subtask": "Merge into final.", "access": [1, 2]}, + ] +} + + +class _JudgeRejectClient(ModelClient): + """Return a scripted generated plan and reject the verifier output.""" + + def __init__(self, plan_text: str) -> None: + super().__init__() + self.plan_text = plan_text + + def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> str: # type: ignore[override] + system = messages[0].get("content", "") if messages else "" + if "workflow conductor" in system: + return self.plan_text + if "verification judge" in system: + return "REJECT" + return "step output" + + +def _generated_orchestrator() -> TaskOrchestrator: + orchestrator = TaskOrchestrator( + [ModelAgent("general_agent", "model-x", tags=("reasoning", "writing", "planning", "research", "verification"))], + client=_JudgeRejectClient(json.dumps(_GENERATED_PLAN)), + ) + orchestrator.policy = replace( + orchestrator.policy, + workflow_planning="generated", + verifier_judge="model", + verifier_required=True, + ) + return orchestrator + + +def test_generated_plan_with_model_judge_rejection_falls_back_to_worker_answer() -> None: + """A required verifier rejection cannot promote synthesizer output.""" + orchestrator = _generated_orchestrator() + result = orchestrator.conduct([{"role": "user", "content": "solve the hard problem"}]) + assert result["plan_source"] == "generated" + assert result["verification"]["accepted"] is False + assert result["verification"]["judge"] == "model" + assert result["answer"] == "step output" + + +def test_parse_workflow_plan_rejects_empty_subtask() -> None: + """Generated workflow steps require a non-empty subtask.""" + orchestrator = _generated_orchestrator() + plan = json.dumps( + { + "steps": [ + {"id": 0, "role": "worker", "agent_id": "general_agent", "subtask": " ", "access": []}, + {"id": 1, "role": "synthesizer", "agent_id": "general_agent", "subtask": "merge", "access": [0]}, + ] + } + ) + with pytest.raises(ValueError) as exc: + orchestrator._parse_workflow_plan(plan) + assert "subtask must be non-empty" in str(exc.value) + + +def test_model_judge_verification_keeps_fallback_when_no_verifier_output() -> None: + """An empty verifier answer leaves the deterministic fallback unchanged.""" + orchestrator = TaskOrchestrator([ModelAgent("general_agent", "m", tags=("reasoning", "verification"))]) + fallback = {"accepted": True, "reason": "term-based", "verifier_output": ""} + assert orchestrator._model_judge_verification("task", fallback) is fallback + + +def test_spend_analytics_reports_mixed_usage_source() -> None: + """Mixed provider-reported and estimated usage is identified explicitly.""" + orchestrator = TaskOrchestrator([ModelAgent("worker_agent", "model-x", tags=("reasoning", "writing"))]) + orchestrator._workflow_runs["run_reported"] = { + "prompt_text": "prompt", + "trace": [{"agent_id": "worker_agent", "output": "aaaa", "usage": {"completion_tokens": 5}}], + } + orchestrator._workflow_runs["run_estimated"] = { + "prompt_text": "prompt", + "trace": [{"agent_id": "worker_agent", "output": "bbbb", "usage": None}], + } + by_model = {row["model"]: row for row in orchestrator.spend_analytics()["by_model"]} + assert by_model["model-x"]["usage_source"] == "mixed" + assert by_model["model-x"]["step_count"] == 2 + + +def test_close_releases_agent_pool_store() -> None: + """Closing an orchestrator releases a configured durable agent pool.""" + with tempfile.TemporaryDirectory() as directory: + orchestrator = TaskOrchestrator( + [ModelAgent("general_agent", "m", tags=("reasoning",))], + agents_db=str(Path(directory) / "pool.sqlite"), + ) + assert orchestrator._pool_store is not None + orchestrator.close() + + +def test_batch_route_blocks_when_budget_exceeded() -> None: + """Batch routing fails closed once the configured output budget is spent.""" + orchestrator = TaskOrchestrator( + [ModelAgent("general_agent", "m", tags=("reasoning", "writing"))], + budget_max_output_tokens=1, + ) + orchestrator.run([{"role": "user", "content": "burn through the tiny token budget right now"}]) + with pytest.raises(BudgetExceededError): + orchestrator.batch_route(["another prompt after the budget is spent"]) + + +def test_batch_route_persists_runs_to_state_store() -> None: + """Batch routing writes completed runs into an explicitly configured store.""" + with tempfile.TemporaryDirectory() as directory: + orchestrator = TaskOrchestrator( + [ModelAgent("general_agent", "m", tags=("reasoning", "writing"))], + budget_max_output_tokens=10_000, + state_db=str(Path(directory) / "state.sqlite"), + ) + assert orchestrator._store is not None + assert orchestrator.budget_status()["exceeded"] is False + records = orchestrator.batch_route(["hello there worker"]) + assert len(records) == 1 + assert len(orchestrator._workflow_runs) == 1 + orchestrator.close() + + +def _orchestrator() -> TaskOrchestrator: + return TaskOrchestrator( + [ + ModelAgent("planner_agent", "mock-planner", tags=("planning", "reasoning")), + ModelAgent("builder_agent", "mock-builder", tags=("coding", "implementation")), + ModelAgent("reviewer_agent", "mock-reviewer", tags=("verification", "security", "review")), + ] + ) + + +def test_is_trace_complete_rejects_empty_and_malformed_traces() -> None: + """Trace completeness fails closed for missing or malformed step evidence.""" + orchestrator = _orchestrator() + assert orchestrator._is_trace_complete({"trace": []}) is False + assert orchestrator._is_trace_complete({"trace": [{"id": 1}]}) is False + assert ( + orchestrator._is_trace_complete( + {"trace": [{"id": 1, "role": "worker", "agent_id": "a", "subtask": "s", "access": "not-a-list", "output": "o"}]} + ) + is False + ) + assert ( + orchestrator._is_trace_complete( + {"trace": [{"id": 1, "role": "worker", "agent_id": "a", "subtask": "s", "access": [], "output": None}]} + ) + is False + ) + + +def test_is_policy_safe_run_rejects_conduct_without_required_verification() -> None: + """Conduct-mode evidence is unsafe when required verification is absent.""" + orchestrator = _orchestrator() + run = {"mode": "conduct", "policy_snapshot": {"verifier_required": True}, "trace": []} + assert orchestrator._is_policy_safe_run(run) is False + + +def test_provider_exclusion_miss_count_counts_unknown_and_excluded_roles() -> None: + """Provider-exclusion evidence counts unresolved agents and disallowed roles.""" + orchestrator = TaskOrchestrator( + [ModelAgent("excluded_agent", "m", "mock://a", tags=("reasoning",), provider_exclusions=("worker",))] + ) + assert orchestrator._provider_exclusion_miss_count({"trace": [{"agent_id": "ghost", "role": "worker"}]}) == 1 + assert orchestrator._provider_exclusion_miss_count({"trace": [{"agent_id": "excluded_agent", "role": "worker"}]}) == 1 + + +def test_security_posture_criterion_fails_on_insecure_profile() -> None: + """An explicitly insecure server profile is a failed security criterion.""" + orchestrator = _orchestrator() + criterion = orchestrator._security_posture_criterion( + { + "auth_mode": "loopback_no_auth", + "allow_public_bind": True, + "expose_trace_by_default": True, + "rate_limit_requests": 0, + "max_concurrent_runs": 0, + } + ) + assert criterion["status"] == "fail" + assert "public bind is enabled" in criterion["evidence"] + + +def test_locale_readiness_criterion_warns_when_keys_missing() -> None: + """Locale bundles with key drift produce an explicit readiness warning.""" + orchestrator = TaskOrchestrator([ModelAgent("general_agent", "m", tags=("reasoning",))]) + analytics = orchestrator.analytics_snapshot(locale_bundles={"en": {"a": "A", "b": "B"}, "ko": {"a": "에이"}}) + criterion = orchestrator._locale_readiness_criterion(analytics) + assert criterion["status"] == "warn" + assert "locale key parity" in criterion["evidence"] + + +def test_provider_egress_criterion_fails_for_insecure_remote_agent() -> None: + """Plain-HTTP remote provider configuration fails commercial egress posture.""" + orchestrator = TaskOrchestrator([ModelAgent("insecure_agent", "gpt-x", "http://api.example.com/v1")]) + criterion = orchestrator._provider_egress_criterion() + assert criterion["status"] == "fail" + assert "insecure_agent" in criterion["evidence"] + + +def test_provider_egress_criterion_accepts_named_credential_over_https() -> None: + """A remote HTTPS provider with a named KV credential passes egress posture.""" + orchestrator = TaskOrchestrator( + [ + ModelAgent( + "secure_agent", + "gpt-x", + "https://api.example.com/v1", + credential_key="PROVIDER_API_KEY", + ) + ] + ) + criterion = orchestrator._provider_egress_criterion() + + assert criterion["status"] == "pass" + assert criterion["evidence"] == "1 remote providers use https and a named KV credential" + + +def test_freeze_report_cache_value_handles_sets_and_unhashables() -> None: + """Cache-key normalization is deterministic for sets and unhashable scalars.""" + assert _freeze_report_cache_value({3, 1, 2}) == (1, 2, 3) + assert _freeze_report_cache_value(bytearray(b"x")) == "bytearray(b'x')" + + +def test_recommend_config_none_and_over_budget_cheapest() -> None: + """Recommendation returns None for no candidates and cheapest if none fit.""" + assert _recommend_config([], None) is None + results = [ + {"name": "quality_config", "quality": 0.9, "cost_usd": 5.0}, + {"name": "budget_config", "quality": 0.5, "cost_usd": 1.0}, + ] + recommendation = _recommend_config(results, cost_budget_usd=0.5) + assert recommendation["name"] == "budget_config" + assert recommendation["reason"] == "no config within budget; cheapest instead" + + +@pytest.mark.parametrize( + ("blocked_count", "warning_count", "expected"), + [(2, 3, "blocked"), (0, 3, "warning"), (0, 0, "ready")], +) +def test_commercial_status_classifier_prioritizes_blockers_then_warnings( + blocked_count: int, + warning_count: int, + expected: str, +) -> None: + """One shared classifier preserves the three public readiness states.""" + assert _classify_commercial_status( + blocked_count, + warning_count, + blocked_status="blocked", + warning_status="warning", + ready_status="ready", + ) == expected + + +_INSECURE_PROFILE = { + "auth_mode": "loopback_no_auth", + "allow_public_bind": True, + "expose_trace_by_default": True, + "rate_limit_requests": 0, + "max_concurrent_runs": 0, +} + +_BLOCKED_REPORTS = [ + ("sales_readiness_report", "readiness_status", "not_ready"), + ("commercial_readiness_report", "commercial_status", "not_commercial_ready"), + ("buyer_evidence_manifest_report", "manifest_status", "buyer_review_blocked"), + ("buyer_handoff_bundle_report", "bundle_status", "buyer_handoff_blocked"), + ("saleability_decision_report", "saleability_status", "saleability_blocked"), + ("commercial_evidence_export_report", "export_status", "commercial_export_blocked"), + ("commercial_acceptance_check_report", "acceptance_status", "commercial_acceptance_blocked"), + ("commercial_release_candidate_report", "release_status", "commercial_release_blocked"), + ("commercial_gap_register_report", "gap_register_status", "commercial_gap_register_blocked"), + ("commercial_procurement_readiness_report", "procurement_status", "commercial_procurement_blocked"), + ("commercial_contract_readiness_report", "contract_status", "commercial_contract_blocked"), + ("commercial_onboarding_readiness_report", "onboarding_status", "commercial_onboarding_blocked"), + ("commercial_operations_readiness_report", "operations_status", "commercial_operations_blocked"), + ("commercial_security_attestation_report", "security_attestation_status", "commercial_security_attestation_blocked"), + ("commercial_value_readiness_report", "value_status", "commercial_value_blocked"), +] + + +def _exercised_orchestrator() -> TaskOrchestrator: + orchestrator = _orchestrator() + orchestrator.record_analytics_event( + "chat_completion_requested", + { + "endpoint_path": "/v1/chat/completions", + "actor_scope": "inference", + "status_code": 200, + "duration_ms": 8, + }, + ) + orchestrator.run( + [{"role": "user", "content": "Analyze the product, implement it, verify it, and summarize."}], + mode="conduct", + ) + orchestrator.run_evaluation(["Replay this readiness prompt."], mode="route") + return orchestrator + + +@pytest.mark.parametrize("method_name,status_field,expected", _BLOCKED_REPORTS) +def test_commercial_reports_classify_blocked_under_insecure_profile( + method_name: str, + status_field: str, + expected: str, +) -> None: + """Commercial evidence reports expose their blocked state under known risk.""" + orchestrator = _exercised_orchestrator() + method = getattr(orchestrator, method_name) + parameters = inspect.signature(method).parameters + kwargs: dict = {} + if "locale_bundles" in parameters: + kwargs["locale_bundles"] = ADMIN_TRANSLATIONS + if "security_profile" in parameters: + kwargs["security_profile"] = _INSECURE_PROFILE + report = method(**kwargs) + assert report[status_field] == expected diff --git a/tests/test_package_metadata.py b/tests/test_package_metadata.py new file mode 100644 index 000000000..d5282b8ab --- /dev/null +++ b/tests/test_package_metadata.py @@ -0,0 +1,212 @@ +"""Verify distribution metadata needed for licensing and buyer due diligence.""" + +import ast +import subprocess +import tarfile +import tomli as tomllib +import zipfile +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + +def has_direct_stdlib_tomllib_import(source: str) -> bool: + """Return whether Python source imports the 3.11-only stdlib parser.""" + + tree = ast.parse(source) + return any( + ( + isinstance(node, ast.Import) + and any( + alias.name == "tomllib" + for alias in node.names + ) + ) + or (isinstance(node, ast.ImportFrom) and node.module == "tomllib") + for node in ast.walk(tree) + ) + + +def test_tomllib_alias_is_rejected_at_the_python_floor() -> None: + """Reject aliased imports of the Python 3.11-only parser.""" + + assert has_direct_stdlib_tomllib_import("import tomllib as parser\n") + + +def discover_test_modules(test_root: Path) -> list[Path]: + """Return modules matching the repository's pytest filename contract.""" + + prefix_named = set(test_root.rglob("test*.py")) + suffix_named = set(test_root.rglob("*_test.py")) + return sorted(prefix_named | suffix_named) + + +def test_pytest_suffix_named_modules_are_scanned(tmp_path: Path) -> None: + """Include pytest's suffix-style module naming convention.""" + + suffix_named_test = tmp_path / "metadata_test.py" + suffix_named_test.write_text("import tomllib as parser\n", encoding="utf-8") + + assert suffix_named_test in discover_test_modules(tmp_path) + + +def test_test_toml_parsers_support_declared_python_floor() -> None: + """Keep every test module importable on the declared minimum Python.""" + + test_paths = discover_test_modules(REPOSITORY_ROOT / "tests") + assert REPOSITORY_ROOT / "tests/fuzz/test_fuzz_properties.py" in test_paths + + for path in test_paths: + assert not has_direct_stdlib_tomllib_import( + path.read_text(encoding="utf-8") + ), path + + this_module = ast.parse(Path(__file__).read_text(encoding="utf-8")) + imports = { + (alias.name, alias.asname) + for node in ast.walk(this_module) + if isinstance(node, ast.Import) + for alias in node.names + } + assert ("tomli", "tomllib") in imports + + +def test_tests_workflow_executes_declared_python_floor() -> None: + """Run the suite on the minimum and current supported interpreters.""" + + workflow = (REPOSITORY_ROOT / ".github/workflows/tests.yml").read_text( + encoding="utf-8" + ) + assert 'python-version: ["3.10", "3.12"]' in workflow + assert "python-version: ${{ matrix.python-version }}" in workflow + + +def packaging_document() -> dict[str, object]: + """Return the parsed packaging configuration from ``pyproject.toml``.""" + + return tomllib.loads( + (REPOSITORY_ROOT / "pyproject.toml").read_text(encoding="utf-8") + ) + + +def project_metadata() -> dict[str, object]: + """Return the static PEP 621 project metadata from ``pyproject.toml``.""" + + return packaging_document()["project"] + + +def test_build_backend_is_exact_and_supports_pep639_metadata() -> None: + """Use one reviewed backend version that understands the license contract.""" + + assert packaging_document()["build-system"] == { + "requires": ["setuptools==83.0.0"], + "build-backend": "setuptools.build_meta", + } + + +def test_distribution_declares_spdx_license_and_includes_license_file() -> None: + """Bind the built distribution to the repository's exact MIT license text.""" + + metadata = project_metadata() + + assert metadata["license"] == "MIT" + assert metadata["license-files"] == ["LICENSE"] + license_text = (REPOSITORY_ROOT / "LICENSE").read_text(encoding="utf-8") + assert license_text.startswith("MIT License\n") + assert "Copyright (c) 2026 ContextualWisdomLab" in license_text + +def test_normal_wheel_and_sdist_carry_pep639_metadata(tmp_path: Path) -> None: + """Build normal artifacts and inspect their emitted licensing authority.""" + + subprocess.run( + ["uv", "build", "--out-dir", str(tmp_path)], + cwd=REPOSITORY_ROOT, + check=True, + capture_output=True, + text=True, + ) + wheels = list(tmp_path.glob("*.whl")) + sdists = list(tmp_path.glob("*.tar.gz")) + assert len(wheels) == 1 + assert len(sdists) == 1 + + expected_license = (REPOSITORY_ROOT / "LICENSE").read_bytes() + expected_headers = { + "License-Expression: MIT", + "License-File: LICENSE", + *{ + f"Project-URL: {label}, {url}" + for label, url in project_metadata()["urls"].items() + }, + } + + with zipfile.ZipFile(wheels[0]) as wheel: + wheel_names = set(wheel.namelist()) + metadata_name = next( + name for name in wheel_names if name.endswith(".dist-info/METADATA") + ) + metadata_headers = set( + wheel.read(metadata_name).decode("utf-8").splitlines() + ) + assert expected_headers <= metadata_headers + license_name = next( + name + for name in wheel_names + if name.endswith(".dist-info/licenses/LICENSE") + ) + assert wheel.read(license_name) == expected_license + + with tarfile.open(sdists[0], mode="r:gz") as sdist: + sdist_names = set(sdist.getnames()) + metadata_name = next( + name for name in sdist_names if name.endswith("/PKG-INFO") + ) + metadata_file = sdist.extractfile(metadata_name) + assert metadata_file is not None + metadata_headers = set( + metadata_file.read().decode("utf-8").splitlines() + ) + assert expected_headers <= metadata_headers + license_name = next( + name + for name in sdist_names + if name.endswith("/LICENSE") and name.count("/") == 1 + ) + license_file = sdist.extractfile(license_name) + assert license_file is not None + assert license_file.read() == expected_license + + changelog = (REPOSITORY_ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + assert "Build and inspect normal wheel and sdist artifacts" in changelog + + +def test_distribution_exposes_authoritative_project_urls() -> None: + """Keep package-registry links anchored to the governed repository.""" + + assert project_metadata()["urls"] == { + "Homepage": "https://github.com/ContextualWisdomLab/contextual-orchestrator", + "Repository": "https://github.com/ContextualWisdomLab/contextual-orchestrator", + "Issues": "https://github.com/ContextualWisdomLab/contextual-orchestrator/issues", + } + + +def test_distribution_description_matches_buyer_facing_product_identity() -> None: + """Describe the governed product instead of the historical lab prototype.""" + + assert project_metadata()["description"] == ( + "Provider-neutral OpenAI-compatible orchestration control plane for " + "governed routing and multi-agent conduct." + ) + + +def test_distribution_metadata_change_is_recorded_for_release_review() -> None: + """Keep the buyer-visible package identity change in release history.""" + + changelog = (REPOSITORY_ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + assert ( + "Declare the MIT SPDX license, packaged license file, authoritative project " + "URLs, and current provider-neutral orchestration-control-plane description " + "in distribution metadata, and pin the PEP 639-capable setuptools build " + "backend." + ) in changelog diff --git a/tests/test_pr_exact_head_workflows.py b/tests/test_pr_exact_head_workflows.py new file mode 100644 index 000000000..7b7fbd2a6 --- /dev/null +++ b/tests/test_pr_exact_head_workflows.py @@ -0,0 +1,30 @@ +"""Contracts that prevent local pull-request workflows from testing stale or synthetic heads.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +EXACT_HEAD_REF = ( + "ref: ${{ github.event_name == 'pull_request' " + "&& github.event.pull_request.head.sha || github.sha }}" +) +WORKFLOW_PATHS = ( + Path(".github/workflows/tests.yml"), + Path(".github/workflows/fuzz.yml"), + Path(".github/workflows/security.yml"), +) + + +@pytest.mark.parametrize("relative_path", WORKFLOW_PATHS) +def test_pull_request_workflows_cover_stacked_exact_heads(relative_path: Path) -> None: + """Require all PR bases to run while every checkout selects the contributor head.""" + workflow = (REPOSITORY_ROOT / relative_path).read_text(encoding="utf-8") + checkout_count = workflow.count("uses: actions/checkout@") + assert checkout_count > 0 + assert "pull_request:\n branches: [main]" not in workflow + assert workflow.count(EXACT_HEAD_REF) == checkout_count + assert workflow.count("persist-credentials: false") >= checkout_count diff --git a/tests/test_provider_address_pinning.py b/tests/test_provider_address_pinning.py new file mode 100644 index 000000000..9c173cf82 --- /dev/null +++ b/tests/test_provider_address_pinning.py @@ -0,0 +1,474 @@ +"""Regression tests for DNS-pinned provider connections.""" + +from __future__ import annotations + +import ssl +import urllib.error +import urllib.request +from unittest import mock + +import pytest + +from contextual_orchestrator import ModelAgent +from contextual_orchestrator.credentials import InMemoryCredentialBackend, set_backend +from contextual_orchestrator.orchestrator import ( + ModelClient, + _literal_loopback_host, +) +from contextual_orchestrator.provider_transport import ( + _PinnedHTTPSConnection, + _ProviderHTTPResponse, + _validated_public_addresses, +) + + +class _FakeResponse: + """Observable provider response double.""" + + def __init__(self, status: int = 200, body: bytes = b"ok") -> None: + """Initialize status, content, headers, and cleanup state.""" + self.status = status + self.reason = "provider status" + self.headers = {"location": "https://attacker.example/v1"} + self.body = body + self.closed = False + + def read(self, *_args: object, **_kwargs: object) -> bytes: + """Return configured response bytes.""" + return self.body + + def close(self) -> None: + """Record response cleanup.""" + self.closed = True + + def __iter__(self): + """Iterate one response line.""" + return iter([self.body]) + + +class _FakeConnection: + """Observable pinned TLS connection double.""" + + created: list["_FakeConnection"] = [] + responses: dict[str, _FakeResponse] = {} + failing_ips: set[str] = set() + + def __init__( + self, + hostname: str, + pinned_ip: str, + port: int, + timeout: float, + context: ssl.SSLContext, + ) -> None: + """Capture construction inputs for transport assertions.""" + self.hostname = hostname + self.pinned_ip = pinned_ip + self.port = port + self.timeout = timeout + self.context = context + self.request_call: tuple[object, ...] | None = None + self.closed = False + self.created.append(self) + + def request( + self, + method: str, + target: str, + body: bytes | None = None, + headers: dict[str, str] | None = None, + ) -> None: + """Capture a request or simulate one address-level failure.""" + self.request_call = (method, target, body, headers) + if self.pinned_ip in self.failing_ips: + raise OSError("address unavailable") + + def getresponse(self) -> _FakeResponse: + """Return the response configured for this address.""" + return self.responses[self.pinned_ip] + + def close(self) -> None: + """Record connection cleanup.""" + self.closed = True + + +@pytest.fixture(autouse=True) +def _reset_credential_backend(): + """Restore the process-global credential backend after every test.""" + try: + yield + finally: + set_backend(None) + + +def _configured_client() -> tuple[ModelClient, ModelAgent]: + """Build a client and HTTPS agent with a resolvable KV credential.""" + backend = InMemoryCredentialBackend() + backend.set("MODEL_KEY", "test-provider-secret") + set_backend(backend) + client = ModelClient(timeout=11) + client._https_connection_class = _FakeConnection + agent = ModelAgent( + "provider_agent", + "provider-model", + "https://api.example.com:8443/v1", + "MODEL_KEY", + ) + _FakeConnection.created = [] + _FakeConnection.responses = {} + _FakeConnection.failing_ips = set() + return client, agent + + +def _public_dns_answers() -> list[tuple[int, int, int, str, tuple[str, int]]]: + """Return duplicate and distinct globally routable IPv4 answers.""" + return [ + (2, 1, 6, "", ("93.184.216.34", 8443)), + (2, 1, 6, "", ("93.184.216.34", 8443)), + (2, 1, 6, "", ("93.184.216.35", 8443)), + ] + + +def test_package_import_keeps_model_client_transport_canonical() -> None: + """Importing the package cannot mutate canonical provider methods.""" + assert ModelClient._validate_provider.__module__ == "contextual_orchestrator.orchestrator" + assert ModelClient._open_provider.__module__ == "contextual_orchestrator.orchestrator" + assert not hasattr(ModelClient, "_dns_pinned_transport_installed") + assert ModelClient()._https_connection_class is _PinnedHTTPSConnection + + +def test_validated_public_addresses_supports_ipv6_and_deduplicates() -> None: + """Address validation returns unique normalized public IPv4 and IPv6 pins.""" + answers = [ + (2, 1, 6, "", ("93.184.216.34", 443)), + (2, 1, 6, "", ("93.184.216.34", 443)), + (10, 1, 6, "", ("2606:2800:220:1:248:1893:25c8:1946", 443, 0, 0)), + ] + with mock.patch( + "contextual_orchestrator.provider_transport.socket.getaddrinfo", + return_value=answers, + ): + assert _validated_public_addresses("api.example.com", 443, "provider_agent") == ( + "93.184.216.34", + "2606:2800:220:1:248:1893:25c8:1946", + ) + + +@pytest.mark.parametrize("unsafe_address", ["127.0.0.1", "100.64.0.1", "224.0.0.1"]) +def test_validated_public_addresses_rejects_unsafe_answer(unsafe_address: str) -> None: + """Any unsafe member of a DNS answer causes validation to fail closed.""" + answers = [ + (2, 1, 6, "", ("93.184.216.34", 443)), + (2, 1, 6, "", (unsafe_address, 443)), + ] + with mock.patch( + "contextual_orchestrator.provider_transport.socket.getaddrinfo", + return_value=answers, + ): + with pytest.raises(RuntimeError, match="non-public address"): + _validated_public_addresses("api.example.com", 443, "provider_agent") + + +def test_validated_public_addresses_rejects_empty_answer() -> None: + """An empty resolver answer cannot silently create an unpinned request.""" + with mock.patch( + "contextual_orchestrator.provider_transport.socket.getaddrinfo", + return_value=[], + ): + with pytest.raises(RuntimeError, match="did not resolve"): + _validated_public_addresses("api.example.com", 443, "provider_agent") + + +def test_validate_then_open_uses_same_dns_answer_without_reresolution() -> None: + """The connected addresses come only from the validation-time DNS answer.""" + client, agent = _configured_client() + with mock.patch( + "contextual_orchestrator.provider_transport.socket.getaddrinfo", + return_value=_public_dns_answers(), + ) as resolver: + client._validate_provider(agent) + assert resolver.call_count == 1 + + _FakeConnection.failing_ips = {"93.184.216.34"} + _FakeConnection.responses = {"93.184.216.35": _FakeResponse(body=b"success")} + request = urllib.request.Request( + "https://api.example.com:8443/v1/chat;mode=fast?trace=yes", + data=b"{}", + headers={"authorization": "Bearer secret"}, + method="POST", + ) + with mock.patch( + "contextual_orchestrator.provider_transport.socket.getaddrinfo", + side_effect=AssertionError("transport must not resolve DNS again"), + ): + with client._open_provider(request) as response: + assert response.read() == b"success" + + first, second = _FakeConnection.created + assert first.closed is True + assert second.hostname == "api.example.com" + assert second.port == 8443 + assert second.timeout == 11 + assert second.request_call == ( + "POST", + "/v1/chat;mode=fast?trace=yes", + b"{}", + {"Authorization": "Bearer secret", "Connection": "close"}, + ) + assert second.closed is True + + +def test_failed_revalidation_clears_existing_pin() -> None: + """A later unsafe DNS answer cannot reuse a formerly valid cached address.""" + client, agent = _configured_client() + with mock.patch( + "contextual_orchestrator.provider_transport.socket.getaddrinfo", + return_value=_public_dns_answers(), + ): + client._validate_provider(agent) + + unsafe = [(2, 1, 6, "", ("127.0.0.1", 8443))] + with mock.patch( + "contextual_orchestrator.provider_transport.socket.getaddrinfo", + return_value=unsafe, + ): + with pytest.raises(RuntimeError, match="non-public address"): + client._validate_provider(agent) + + request = urllib.request.Request("https://api.example.com:8443/v1/chat") + with pytest.raises(RuntimeError, match="no validated address pin"): + client._open_provider(request) + + +def test_redirect_response_is_rejected_without_following_location() -> None: + """A redirect cannot forward provider credentials to another destination.""" + client, agent = _configured_client() + answers = [(2, 1, 6, "", ("93.184.216.34", 8443))] + with mock.patch( + "contextual_orchestrator.provider_transport.socket.getaddrinfo", + return_value=answers, + ): + client._validate_provider(agent) + + response = _FakeResponse(status=302) + _FakeConnection.responses = {"93.184.216.34": response} + with pytest.raises(urllib.error.HTTPError) as exc_info: + client._open_provider( + urllib.request.Request("https://api.example.com:8443/v1/chat", method="POST") + ) + assert exc_info.value.code == 302 + assert response.closed is True + assert _FakeConnection.created[0].closed is True + assert len(_FakeConnection.created) == 1 + + +def test_all_pinned_addresses_failing_returns_network_error() -> None: + """Exhausting all approved addresses yields one urllib-compatible error.""" + client, agent = _configured_client() + with mock.patch( + "contextual_orchestrator.provider_transport.socket.getaddrinfo", + return_value=_public_dns_answers(), + ): + client._validate_provider(agent) + + _FakeConnection.failing_ips = {"93.184.216.34", "93.184.216.35"} + with pytest.raises(urllib.error.URLError, match="address unavailable"): + client._open_provider(urllib.request.Request("https://api.example.com:8443", method="GET")) + assert all(connection.closed for connection in _FakeConnection.created) + + +def test_open_provider_rejects_unsupported_scheme() -> None: + """The low-level transport independently rejects non-HTTP provider schemes.""" + client = ModelClient() + with pytest.raises(RuntimeError, match=r"http\(s\)"): + client._open_provider(urllib.request.Request("file:///etc/passwd")) + + +def test_provider_response_delegates_metadata_iteration_read_and_cleanup() -> None: + """The wrapper preserves response behavior and always closes its connection.""" + response = mock.Mock() + response.status = 200 + response.read.return_value = b"payload" + response.__iter__ = mock.Mock(return_value=iter([b"one", b"two"])) + connection = mock.Mock() + wrapper = _ProviderHTTPResponse(response, connection) + with wrapper as entered: + assert entered is wrapper + assert wrapper.status == 200 + assert wrapper.read(4) == b"payload" + assert list(wrapper) == [b"one", b"two"] + response.read.assert_called_once_with(4) + response.close.assert_called_once_with() + connection.close.assert_called_once_with() + + +def test_provider_response_closes_connection_when_response_close_fails() -> None: + """Connection cleanup survives an exception from response cleanup.""" + response = mock.Mock() + response.close.side_effect = RuntimeError("close failed") + connection = mock.Mock() + wrapper = _ProviderHTTPResponse(response, connection) + with pytest.raises(RuntimeError, match="close failed"): + wrapper.close() + connection.close.assert_called_once_with() + + +def test_pinned_https_connection_dials_ip_and_preserves_sni() -> None: + """The direct socket uses the pin while TLS verifies the original hostname.""" + raw_socket = mock.Mock() + wrapped_socket = object() + context = mock.Mock() + context.wrap_socket.return_value = wrapped_socket + with mock.patch( + "contextual_orchestrator.provider_transport.socket.create_connection", + return_value=raw_socket, + ) as create_connection: + connection = _PinnedHTTPSConnection( + "api.example.com", + "93.184.216.34", + 443, + 7.0, + context, + ) + connection.connect() + create_connection.assert_called_once_with(("93.184.216.34", 443), 7.0, None) + context.wrap_socket.assert_called_once_with(raw_socket, server_hostname="api.example.com") + assert connection.sock is wrapped_socket + + +def test_pinned_https_connection_closes_socket_when_tls_setup_fails() -> None: + """A TLS setup failure cannot leak the already-connected raw socket.""" + raw_socket = mock.Mock() + context = mock.Mock() + context.wrap_socket.side_effect = ssl.SSLError("handshake failed") + with mock.patch( + "contextual_orchestrator.provider_transport.socket.create_connection", + return_value=raw_socket, + ): + connection = _PinnedHTTPSConnection( + "api.example.com", + "93.184.216.34", + 443, + 7.0, + context, + ) + with pytest.raises(ssl.SSLError, match="handshake failed"): + connection.connect() + raw_socket.close.assert_called_once_with() + + +@pytest.mark.parametrize( + ("hostname", "expected"), + [ + (None, False), + ("localhost", True), + ("localhost.", True), + ("127.0.0.1", True), + ("::1", True), + ("192.0.2.1", False), + ("api.example.com", False), + ("localhost.example", False), + ], +) +def test_literal_loopback_host_classification( + hostname: str | None, + expected: bool, +) -> None: + """Only localhost and literal loopback addresses enter the HTTP test seam.""" + assert _literal_loopback_host(hostname) is expected + + +@pytest.mark.parametrize( + "url", + [ + "http://api.example.com/v1/chat", + "http://192.0.2.1/v1/chat", + ], +) +def test_open_provider_rejects_non_loopback_http(url: str) -> None: + """Direct low-level callers cannot use plain HTTP outside loopback.""" + client = ModelClient() + with pytest.raises(RuntimeError, match="literal loopback"): + client._open_provider(urllib.request.Request(url, method="POST")) + + +def test_open_provider_rejects_url_userinfo() -> None: + """Provider URLs cannot smuggle credentials through URL user information.""" + client = ModelClient() + request = urllib.request.Request( + "http://user:password@127.0.0.1:8080/v1/chat", + method="POST", + ) + with pytest.raises(RuntimeError, match="user information"): + client._open_provider(request) + + +def test_loopback_http_uses_direct_connection_and_bypasses_proxy( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The integration seam connects directly and ignores ambient proxy state.""" + client = ModelClient(timeout=13) + response = _FakeResponse(body=b"loopback") + connection = mock.Mock() + connection.getresponse.return_value = response + connection_class = mock.Mock(return_value=connection) + client._http_connection_class = connection_class + monkeypatch.setenv("HTTP_PROXY", "http://proxy.example:3128") + monkeypatch.setenv("NO_PROXY", "") + request = urllib.request.Request( + "http://127.0.0.1:8080/v1/chat?trace=yes", + data=b"{}", + headers={"authorization": "Bearer local-secret"}, + method="POST", + ) + + with mock.patch( + "contextual_orchestrator.orchestrator.urllib.request.urlopen", + side_effect=AssertionError("ambient proxy-capable opener must not run"), + ) as urlopen: + with client._open_provider(request) as opened: + assert opened.read() == b"loopback" + + urlopen.assert_not_called() + connection_class.assert_called_once_with("127.0.0.1", 8080, timeout=13) + connection.request.assert_called_once_with( + "POST", + "/v1/chat?trace=yes", + body=b"{}", + headers={"Authorization": "Bearer local-secret", "Connection": "close"}, + ) + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_loopback_http_rejects_redirect_and_closes_resources() -> None: + """A loopback response cannot redirect credentials to another origin.""" + client = ModelClient() + response = _FakeResponse(status=302) + connection = mock.Mock() + connection.getresponse.return_value = response + client._http_connection_class = mock.Mock(return_value=connection) + + with pytest.raises(urllib.error.HTTPError) as exc_info: + client._open_provider( + urllib.request.Request("http://localhost:8080/v1/chat", method="POST") + ) + + assert exc_info.value.code == 302 + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_loopback_http_connection_failure_closes_and_is_transient() -> None: + """A failed direct loopback connection is closed and surfaced as URLError.""" + client = ModelClient() + connection = mock.Mock() + connection.request.side_effect = OSError("loopback unavailable") + client._http_connection_class = mock.Mock(return_value=connection) + + with pytest.raises(urllib.error.URLError, match="loopback unavailable"): + client._open_provider( + urllib.request.Request("http://[::1]:8080/v1/chat", method="POST") + ) + + connection.close.assert_called_once_with() diff --git a/tests/test_provider_catalog.py b/tests/test_provider_catalog.py new file mode 100644 index 000000000..fa55eb015 --- /dev/null +++ b/tests/test_provider_catalog.py @@ -0,0 +1,415 @@ +"""Contracts for durable multi-provider discovery, bootstrap, and routing.""" + +from __future__ import annotations + +from dataclasses import replace +import json +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + get_credential, + set_backend, +) +from contextual_orchestrator.provider_catalog import ( # noqa: E402 + DEFAULT_PROVIDER_ACCOUNTS, + PROVIDER_CATALOG_SCHEMA_SQL, + CatalogHttpError, + DiscoveredModel, + InMemoryProviderCatalogStore, + ProviderAwareModelClient, + ProviderCatalogHttpClient, + ProviderCatalogService, + ProviderCatalogUnavailable, + bootstrap_provider_credentials, + build_catalog_orchestrator, + normalize_models_document, +) +from contextual_orchestrator.orchestrator import ModelAgent # noqa: E402 + + +@pytest.fixture(autouse=True) +def _isolated_credentials(): + """Keep every provider bootstrap test isolated from ambient credentials.""" + set_backend(InMemoryCredentialBackend()) + try: + yield + finally: + set_backend(None) + + +def _models(*names: str) -> list[DiscoveredModel]: + """Build deterministic model fixtures for one provider account.""" + return [ + DiscoveredModel( + model_name=name, + display_name=name, + capabilities=("chat", "reasoning"), + modalities=("text",), + context_window=131_072, + input_price_usd_per_million=1.0, + output_price_usd_per_million=2.0, + ) + for name in names + ] + + +def test_default_accounts_cover_every_configured_secret_and_split_nvidia_accounts() -> None: + """The built-in catalog maps all five GitHub secret names without collapsing NIM keys.""" + credential_names = [account.credential_name for account in DEFAULT_PROVIDER_ACCOUNTS] + assert credential_names == [ + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "BYTEZ_API_KEY", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", + ] + account_ids = {account.provider_account_id for account in DEFAULT_PROVIDER_ACCOUNTS} + assert account_ids == { + "nvidia_nim_primary", + "nvidia_nim_secondary", + "bytez_primary", + "openrouter_primary", + "openai_primary", + } + + +def test_bootstrap_registers_all_credentials_without_returning_values() -> None: + """One-shot environment transport writes every secret into KV and reports names only.""" + environment = { + account.credential_name: f"secret-{index}-value" + for index, account in enumerate(DEFAULT_PROVIDER_ACCOUNTS) + } + + summary = bootstrap_provider_credentials(environment, require_all=True) + + assert summary == { + "registered_credentials": [account.credential_name for account in DEFAULT_PROVIDER_ACCOUNTS], + "missing_credentials": [], + } + for account in DEFAULT_PROVIDER_ACCOUNTS: + assert get_credential(account.credential_name) == environment[account.credential_name] + assert "secret-" not in json.dumps(summary) + + +def test_bootstrap_fails_closed_before_partial_write_when_required_secret_is_missing() -> None: + """Required bootstrap validates the complete fixed inventory before mutating KV.""" + environment = { + account.credential_name: "configured-value" + for account in DEFAULT_PROVIDER_ACCOUNTS[:-1] + } + + with pytest.raises(ProviderCatalogUnavailable, match="inventory is incomplete"): + bootstrap_provider_credentials(environment, require_all=True) + + assert all(get_credential(account.credential_name) is None for account in DEFAULT_PROVIDER_ACCOUNTS) + + +def test_optional_bootstrap_registers_present_credentials_and_reports_missing_names() -> None: + """Non-production bootstrap may seed a subset while keeping missing names explicit.""" + first, second = DEFAULT_PROVIDER_ACCOUNTS[:2] + summary = bootstrap_provider_credentials( + {first.credential_name: "primary-value"}, + require_all=False, + accounts=(first, second), + ) + assert summary == { + "registered_credentials": [first.credential_name], + "missing_credentials": [second.credential_name], + } + + +def test_models_document_normalizes_openai_shape_and_rejects_invalid_rows() -> None: + """OpenAI-compatible listings become bounded provider-neutral model records.""" + models = normalize_models_document( + { + "data": [ + { + "id": "alpha/reasoner", + "context_length": 200_000, + "pricing": {"prompt": "0.000001", "completion": "0.000002"}, + }, + { + "id": "vision-model", + "architecture": { + "input_modalities": ["text", "image"], + "output_modalities": ["text"], + }, + }, + {"id": ""}, + {"object": "model"}, + 42, + ] + } + ) + + assert [model.model_name for model in models] == ["alpha/reasoner", "vision-model"] + assert models[0].context_window == 200_000 + assert models[0].input_price_usd_per_million == pytest.approx(1.0) + assert models[0].output_price_usd_per_million == pytest.approx(2.0) + assert "reasoning" in models[0].capabilities + assert "vision" in models[1].capabilities + assert models[1].modalities == ("image", "text") + + +def test_models_document_accepts_models_mapping_and_string_rows() -> None: + """Provider-specific mapping and string inventories normalize without adapter branching.""" + models = normalize_models_document({"models": {"first": "plain-model", "second": {"name": "embed-model"}}}) + assert [model.model_name for model in models] == ["embed-model", "plain-model"] + assert models[0].capabilities == ("embeddings",) + assert models[1].capabilities == ("chat",) + + +def test_models_document_rejects_malformed_root_and_unsafe_numeric_metadata() -> None: + """Malformed roots and non-finite/negative metadata never enter routing evidence.""" + assert normalize_models_document({"data": "not-a-list"}) == [] + model = normalize_models_document( + { + "data": [ + { + "id": "safe-model", + "context_length": -1, + "pricing": {"prompt": "nan", "completion": "-1"}, + } + ] + } + )[0] + assert model.context_window is None + assert model.input_price_usd_per_million is None + assert model.output_price_usd_per_million is None + + +def test_http_client_retries_transient_failure_with_bounded_backoff() -> None: + """Transient catalog errors retry, while the successful document is normalized once.""" + sleeps: list[float] = [] + client = ProviderCatalogHttpClient( + max_attempts=2, + sleep=sleeps.append, + random_uniform=lambda _low, high: high, + ) + calls: list[int] = [] + + def fake_request(_account, _credential): + calls.append(1) + if len(calls) == 1: + raise CatalogHttpError("catalog_http_503", transient=True) + return {"data": [{"id": "recovered-model"}]} + + client._request_json = fake_request # type: ignore[method-assign] + models = client.discover(DEFAULT_PROVIDER_ACCOUNTS[0], "credential") + assert [model.model_name for model in models] == ["recovered-model"] + assert len(calls) == 2 + assert sleeps == [0.5] + + +def test_http_client_does_not_retry_permanent_or_empty_catalog() -> None: + """Authentication and structurally empty catalogs fail fast with stable codes.""" + client = ProviderCatalogHttpClient(max_attempts=3, sleep=lambda _delay: None) + client._request_json = lambda _account, _credential: (_ for _ in ()).throw( # type: ignore[method-assign] + CatalogHttpError("catalog_authentication_failed") + ) + with pytest.raises(CatalogHttpError, match="catalog_authentication_failed"): + client.discover(DEFAULT_PROVIDER_ACCOUNTS[0], "credential") + + client._request_json = lambda _account, _credential: {"data": []} # type: ignore[method-assign] + with pytest.raises(CatalogHttpError, match="catalog_contains_no_models"): + client.discover(DEFAULT_PROVIDER_ACCOUNTS[0], "credential") + + +def test_refresh_isolates_provider_failure_and_preserves_last_known_good_catalog() -> None: + """A failed account refresh cannot erase its prior usable model set or stop peers.""" + store = InMemoryProviderCatalogStore() + primary, secondary = DEFAULT_PROVIDER_ACCOUNTS[:2] + store.replace_catalog(primary, _models("nim-primary-old")) + store.replace_catalog(secondary, _models("nim-secondary-old")) + bootstrap_provider_credentials( + { + primary.credential_name: "primary-secret", + secondary.credential_name: "secondary-secret", + }, + require_all=False, + accounts=(primary, secondary), + ) + + def discover(account, _credential): + if account.provider_account_id == primary.provider_account_id: + raise CatalogHttpError("provider_unavailable", transient=True) + return _models("nim-secondary-new") + + service = ProviderCatalogService(store=store, accounts=(primary, secondary), discover=discover) + summary = service.refresh_all() + + assert summary["provider_accounts"][primary.provider_account_id]["status"] == "stale_available" + assert summary["provider_accounts"][secondary.provider_account_id]["status"] == "refreshed" + enabled = {(row.provider_account_id, row.model.model_name) for row in store.enabled_models()} + assert (primary.provider_account_id, "nim-primary-old") in enabled + assert (secondary.provider_account_id, "nim-secondary-new") in enabled + assert (secondary.provider_account_id, "nim-secondary-old") not in enabled + + +def test_refresh_classifies_missing_credentials_disabled_accounts_and_adapter_failures() -> None: + """Account-local configuration and unexpected adapter exceptions remain explicit.""" + first = DEFAULT_PROVIDER_ACCOUNTS[0] + disabled = replace(DEFAULT_PROVIDER_ACCOUNTS[1], enabled=False) + third = DEFAULT_PROVIDER_ACCOUNTS[2] + bootstrap_provider_credentials({third.credential_name: "configured"}, require_all=False, accounts=(third,)) + service = ProviderCatalogService( + store=InMemoryProviderCatalogStore(), + accounts=(first, disabled, third), + discover=lambda _account, _credential: (_ for _ in ()).throw(RuntimeError("private detail")), + ) + with pytest.raises(ProviderCatalogUnavailable): + service.refresh_all() + rows = service.last_refresh_summary["provider_accounts"] + assert rows[first.provider_account_id]["error_code"] == "credential_not_registered" + assert rows[disabled.provider_account_id]["status"] == "disabled" + assert rows[third.provider_account_id]["error_code"] == "catalog_adapter_failure" + assert "private detail" not in json.dumps(rows) + + +def test_refresh_raises_only_when_no_fresh_or_last_known_good_candidate_exists() -> None: + """An empty first bootstrap fails loudly instead of starting with a mock or empty pool.""" + account = DEFAULT_PROVIDER_ACCOUNTS[0] + bootstrap_provider_credentials({account.credential_name: "secret"}, require_all=False, accounts=(account,)) + service = ProviderCatalogService( + store=InMemoryProviderCatalogStore(), + accounts=(account,), + discover=lambda _account, _credential: (_ for _ in ()).throw( + CatalogHttpError("provider_unavailable", transient=True) + ), + ) + with pytest.raises(ProviderCatalogUnavailable, match="no usable provider model"): + service.refresh_all() + + +def test_catalog_builds_distinct_agents_and_keeps_credential_names_not_values() -> None: + """Discovered models become valid ModelAgent rows with provider-account isolation.""" + store = InMemoryProviderCatalogStore() + primary, secondary = DEFAULT_PROVIDER_ACCOUNTS[:2] + store.replace_catalog(primary, _models("shared-model")) + store.replace_catalog(secondary, _models("shared-model")) + + agents = ProviderCatalogService(store=store, accounts=(primary, secondary)).candidate_agents() + + assert len(agents) == 2 + assert len({agent.id for agent in agents}) == 2 + assert {agent.credential_key for agent in agents} == { + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + } + assert {agent.provider_name for agent in agents} == {"nvidia_nim"} + assert all("secret" not in json.dumps(agent.to_config()).lower() for agent in agents) + + +def test_catalog_orchestrator_uses_role_tags_and_retains_cross_provider_failover() -> None: + """The paper-grounded orchestrator receives role-capable candidates from distinct providers.""" + store = InMemoryProviderCatalogStore() + reasoning_account = DEFAULT_PROVIDER_ACCOUNTS[0] + coding_account = DEFAULT_PROVIDER_ACCOUNTS[3] + store.replace_catalog( + reasoning_account, + [DiscoveredModel("deep-reasoner", "Deep Reasoner", ("chat", "reasoning"), ("text",), 200_000)], + ) + store.replace_catalog( + coding_account, + [DiscoveredModel("code-specialist", "Code Specialist", ("chat", "coding"), ("text",), 128_000)], + ) + + orchestrator = build_catalog_orchestrator(store, accounts=(reasoning_account, coding_account)) + + assert len(orchestrator.agents) == 2 + assert orchestrator._select_agent("plan and analyze", "thinker").model == "deep-reasoner" + assert orchestrator._select_agent("implement this code", "worker").model == "code-specialist" + assert len(orchestrator._failover_candidates(orchestrator.agents[0], "verify", "verifier")) == 2 + + +def test_disabled_provider_account_is_excluded_without_deleting_catalog_history() -> None: + """Governance can disable an account while retaining its catalog and refresh evidence.""" + store = InMemoryProviderCatalogStore() + account = DEFAULT_PROVIDER_ACCOUNTS[0] + store.replace_catalog(account, _models("candidate-model")) + disabled = replace(account, enabled=False) + store.upsert_account(disabled) + + assert ProviderCatalogService(store=store, accounts=(disabled,)).candidate_agents() == [] + assert len(store.all_models()) == 1 + + +def test_bytez_client_uses_native_key_transport_and_normalizes_output() -> None: + """Bytez candidates use their native contract instead of a fabricated OpenAI bearer call.""" + captured: list[tuple[ModelAgent, list[dict[str, str]], str]] = [] + + def bytez_request(agent, messages, credential): + captured.append((agent, messages, credential)) + return {"output": {"content": "bytez-answer"}} + + client = ProviderAwareModelClient(bytez_request=bytez_request) + agent = ModelAgent( + "bytez_worker", + "owner/model", + "https://api.bytez.com", + credential_key="BYTEZ_API_KEY", + provider_name="bytez", + ) + bootstrap_provider_credentials({"BYTEZ_API_KEY": "bytez-secret"}, require_all=False, accounts=(DEFAULT_PROVIDER_ACCOUNTS[2],)) + + answer = client.chat(agent, [{"role": "user", "content": "hello"}]) + + assert answer == "bytez-answer" + assert captured[0][2] == "bytez-secret" + assert client.take_usage() is None + + +def test_bytez_client_fails_closed_on_missing_credential_or_unsupported_output() -> None: + """Native transport never sends an empty key or accepts an ambiguous provider result.""" + agent = ModelAgent( + "bytez_worker", + "owner/model", + "https://api.bytez.com", + credential_key="BYTEZ_API_KEY", + provider_name="bytez", + ) + client = ProviderAwareModelClient(bytez_request=lambda _agent, _messages, _credential: {"output": []}) + with pytest.raises(ProviderCatalogUnavailable, match="credential is not registered"): + client.chat(agent, [{"role": "user", "content": "hello"}]) + + bootstrap_provider_credentials({"BYTEZ_API_KEY": "secret"}, require_all=False, accounts=(DEFAULT_PROVIDER_ACCOUNTS[2],)) + with pytest.raises(ProviderCatalogUnavailable, match="response shape is unsupported"): + client.chat(agent, [{"role": "user", "content": "hello"}]) + + empty_mapping_client = ProviderAwareModelClient( + bytez_request=lambda _agent, _messages, _credential: {"output": {}} + ) + with pytest.raises(ProviderCatalogUnavailable, match="response shape is unsupported"): + empty_mapping_client.chat(agent, [{"role": "user", "content": "hello"}]) + + +def test_non_bytez_client_delegates_to_existing_model_client_mock_path() -> None: + """Provider awareness leaves the existing mock/OpenAI-compatible behavior unchanged.""" + client = ProviderAwareModelClient() + agent = ModelAgent("general_agent", "mock-generalist", "mock://local") + assert client.chat(agent, [{"role": "user", "content": "hello"}]) + + +def test_schema_is_normalized_and_never_stores_provider_secret_values() -> None: + """The production catalog DDL keeps credentials referenced by name in normalized tables.""" + normalized = " ".join(PROVIDER_CATALOG_SCHEMA_SQL.lower().split()) + for table_name in ( + "provider_accounts", + "provider_models", + "model_capabilities", + "model_modalities", + "catalog_refresh_runs", + ): + assert f"create table if not exists {table_name}" in normalized + assert "references provider_accounts" in normalized + assert "references provider_models" in normalized + assert "credential_name" in normalized + assert "secret_value" not in normalized + assert "api_key_value" not in normalized + assert "encrypted_value" not in normalized diff --git a/tests/test_provider_catalog_cli.py b/tests/test_provider_catalog_cli.py new file mode 100644 index 000000000..dadbdf57e --- /dev/null +++ b/tests/test_provider_catalog_cli.py @@ -0,0 +1,155 @@ +"""CLI wiring for catalog-backed provider discovery and runtime startup.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import contextual_orchestrator.__main__ as cli # noqa: E402 +from contextual_orchestrator import ModelAgent # noqa: E402 +from contextual_orchestrator.provider_catalog import ProviderCatalogUnavailable # noqa: E402 + + +class _Parser: + """Small parser double that records fail-closed usage errors.""" + + def error(self, message: str) -> None: + """Raise a deterministic exception carrying the parser error text.""" + raise ValueError(message) + + +class _RuntimeOrchestrator: + """CLI-facing orchestrator double for catalog startup tests.""" + + instances: list["_RuntimeOrchestrator"] = [] + + def __init__(self, agents, **kwargs) -> None: + self.agents = agents + self.kwargs = kwargs + self.complete_calls: list[tuple[list[dict[str, str]], str]] = [] + type(self).instances.append(self) + + def complete(self, messages, mode="auto"): + """Record one completion and return a deterministic response.""" + self.complete_calls.append((messages, mode)) + return {"answer": "catalog-answer", "mode": mode} + + def compare_to_baseline(self, prompts, mode="auto"): + """Return a deterministic evaluation response for interface completeness.""" + return {"prompts": prompts, "mode": mode} + + +def _catalog_agent() -> ModelAgent: + """Return one valid discovered agent fixture.""" + return ModelAgent( + "openai_catalog_agent", + "catalog-model", + "https://api.openai.com/v1", + credential_key="OPENAI_API_KEY", + provider_name="openai", + ) + + +def test_runtime_agents_uses_seed_loader_without_catalog(monkeypatch: pytest.MonkeyPatch) -> None: + """The explicit seed-file path remains unchanged when no durable DSN is selected.""" + expected = [_catalog_agent()] + monkeypatch.setattr(cli, "load_agents", lambda path: expected if path == "agents.json" else []) + args = argparse.Namespace(provider_catalog_dsn=None, agents="agents.json") + assert cli._runtime_agents(_Parser(), args) is expected + + +def test_runtime_agents_loads_catalog_candidates(monkeypatch: pytest.MonkeyPatch) -> None: + """A configured durable DSN replaces the seed file with enabled catalog models.""" + expected = [_catalog_agent()] + stores: list[str] = [] + + class _Store: + def __init__(self, dsn: str) -> None: + stores.append(dsn) + + class _Service: + def __init__(self, *, store) -> None: + self.store = store + + def candidate_agents(self): + return expected + + monkeypatch.setattr(cli, "PostgresProviderCatalogStore", _Store) + monkeypatch.setattr(cli, "ProviderCatalogService", _Service) + args = argparse.Namespace(provider_catalog_dsn="postgresql://catalog", agents="ignored.json") + + assert cli._runtime_agents(_Parser(), args) is expected + assert stores == ["postgresql://catalog"] + + +def test_runtime_agents_reports_catalog_initialization_failure(monkeypatch: pytest.MonkeyPatch) -> None: + """Durable catalog errors reach argparse without a silent seed or memory fallback.""" + monkeypatch.setattr( + cli, + "PostgresProviderCatalogStore", + lambda _dsn: (_ for _ in ()).throw(ProviderCatalogUnavailable("catalog unavailable")), + ) + args = argparse.Namespace(provider_catalog_dsn="postgresql://catalog", agents="ignored.json") + with pytest.raises(ValueError, match="catalog unavailable"): + cli._runtime_agents(_Parser(), args) + + +def test_runtime_agents_rejects_empty_catalog(monkeypatch: pytest.MonkeyPatch) -> None: + """An initialized but empty catalog cannot fall back to the bundled mock pool.""" + monkeypatch.setattr(cli, "PostgresProviderCatalogStore", lambda _dsn: object()) + + class _Service: + def __init__(self, *, store) -> None: + self.store = store + + def candidate_agents(self): + return [] + + monkeypatch.setattr(cli, "ProviderCatalogService", _Service) + args = argparse.Namespace(provider_catalog_dsn="postgresql://catalog", agents="ignored.json") + with pytest.raises(ValueError, match="no enabled candidates"): + cli._runtime_agents(_Parser(), args) + + +def test_main_catalog_mode_uses_provider_aware_client( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Catalog mode wires the provider-aware client into the ordinary orchestrator.""" + _RuntimeOrchestrator.instances.clear() + agent = _catalog_agent() + client_calls: list[dict[str, object]] = [] + monkeypatch.setattr(cli, "_runtime_agents", lambda _parser, _args: [agent]) + monkeypatch.setattr( + cli, + "ProviderAwareModelClient", + lambda **kwargs: client_calls.append(kwargs) or {"provider_client": kwargs}, + ) + monkeypatch.setattr(cli, "TaskOrchestrator", _RuntimeOrchestrator) + monkeypatch.setattr( + sys, + "argv", + [ + "contextual-orchestrator", + "catalog prompt", + "--provider-catalog-dsn", + "postgresql://catalog", + "--mode", + "route", + ], + ) + + cli.main() + + instance = _RuntimeOrchestrator.instances[-1] + assert instance.agents == [agent] + assert instance.kwargs["client"]["provider_client"]["verify_tls"] is True + assert client_calls == [{"ca_bundle": None, "verify_tls": True}] + assert instance.complete_calls == [([{"role": "user", "content": "catalog prompt"}], "route")] + assert json.loads(capsys.readouterr().out) == {"answer": "catalog-answer", "mode": "route"} diff --git a/tests/test_provider_catalog_coverage.py b/tests/test_provider_catalog_coverage.py new file mode 100644 index 000000000..3e3351d7d --- /dev/null +++ b/tests/test_provider_catalog_coverage.py @@ -0,0 +1,230 @@ +"""Focused branch coverage for provider-catalog boundary helpers.""" + +from __future__ import annotations + +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import contextual_orchestrator.provider_catalog as catalog # noqa: E402 +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + set_backend, +) +from contextual_orchestrator.orchestrator import ModelAgent # noqa: E402 + + +@pytest.fixture(autouse=True) +def _credential_backend(): + """Use one isolated credential registry for every focused branch test.""" + set_backend(InMemoryCredentialBackend()) + try: + yield + finally: + set_backend(None) + + +def test_provider_account_can_explicitly_disable_catalog_discovery() -> None: + """An account without a listing endpoint advertises no models URL.""" + account = catalog.ProviderAccount( + "custom_provider", + "custom_provider", + "CUSTOM_PROVIDER_KEY", + "https://models.example", + models_path=None, + ) + assert account.models_url is None + client = catalog.ProviderCatalogHttpClient() + with pytest.raises(catalog.CatalogHttpError, match="catalog_endpoint_not_configured"): + client.discover(account, "credential") + + +def test_http_limit_validation_and_deadline_failure() -> None: + """Invalid limits and an exhausted wall-clock deadline fail before network access.""" + for options in ( + {"timeout_seconds": 0}, + {"max_attempts": 0}, + {"deadline_seconds": 0}, + ): + with pytest.raises(ValueError, match="limits must be positive"): + catalog.ProviderCatalogHttpClient(**options) + + ticks = iter((10.0, 11.0)) + client = catalog.ProviderCatalogHttpClient(deadline_seconds=0.5, clock=lambda: next(ticks)) + with pytest.raises(catalog.CatalogHttpError, match="catalog_deadline_exceeded"): + client.discover(catalog.DEFAULT_PROVIDER_ACCOUNTS[0], "credential") + + +def test_http_attempts_exhausted_guard_is_stable(monkeypatch: pytest.MonkeyPatch) -> None: + """The defensive post-loop guard retains a stable secret-free error code.""" + client = catalog.ProviderCatalogHttpClient() + monkeypatch.setattr(catalog, "range", lambda _count: [], raising=False) + with pytest.raises(catalog.CatalogHttpError, match="catalog_attempts_exhausted"): + client.discover(catalog.DEFAULT_PROVIDER_ACCOUNTS[0], "credential") + + +def test_model_normalization_covers_specialized_capabilities_and_bad_values() -> None: + """Reranking, moderation, audio, guard, and malformed metadata stay deterministic.""" + models = catalog.normalize_models_document( + { + "data": [ + {"id": "rank/rerank-large"}, + {"id": "safe/moderation-latest"}, + {"id": "voice/whisper-audio", "modalities": "audio"}, + {"id": "secure/guard-model"}, + { + "id": "invalid-metadata", + "context_length": object(), + "pricing": {"prompt": object(), "completion": True}, + "capabilities": ["", 7, "x" * 129, "CUSTOM"], + }, + {"id": "code/coder-model"}, + {"id": "x" * 513}, + ] + } + ) + by_name = {model.model_name: model for model in models} + assert by_name["rank/rerank-large"].capabilities == ("reranking",) + assert by_name["safe/moderation-latest"].capabilities == ("moderation",) + assert by_name["voice/whisper-audio"].capabilities == ("audio", "chat") + assert by_name["secure/guard-model"].capabilities == ("chat", "moderation") + invalid = by_name["invalid-metadata"] + assert invalid.capabilities == ("chat", "custom") + assert invalid.context_window is None + assert invalid.input_price_usd_per_million is None + assert invalid.output_price_usd_per_million is None + assert by_name["code/coder-model"].capabilities == ("chat", "coding") + assert len(by_name) == 6 + + +def test_candidate_tags_cover_multimodal_and_empty_slug_fallback() -> None: + """Multimodal role tags and hostile model identifiers produce valid agent records.""" + account = catalog.DEFAULT_PROVIDER_ACCOUNTS[4] + store = catalog.InMemoryProviderCatalogStore() + store.replace_catalog( + account, + [ + catalog.DiscoveredModel( + model_name="!!!", + display_name="Punctuation", + capabilities=("chat", "coding", "vision", "audio"), + modalities=("audio", "image", "text"), + context_window=300_000, + input_price_usd_per_million=0.0, + output_price_usd_per_million=0.0, + ) + ], + ) + agent = catalog.ProviderCatalogService(store=store, accounts=(account,)).candidate_agents()[0] + assert "model_worker" in agent.id + assert {"implementation", "debugging", "image", "speech", "multimodal"}.issubset(agent.tags) + assert agent.priority == account.priority_rank + 5 + + +def test_empty_catalog_factory_fails_closed() -> None: + """Runtime construction never falls back to an implicit mock worker.""" + with pytest.raises(catalog.ProviderCatalogUnavailable, match="no enabled agents"): + catalog.build_catalog_orchestrator(catalog.InMemoryProviderCatalogStore()) + + +def test_refresh_rejects_empty_discovery_without_fabricating_candidates() -> None: + """An empty provider response is recorded and cannot bootstrap an agent pool.""" + account = catalog.DEFAULT_PROVIDER_ACCOUNTS[0] + catalog.bootstrap_provider_credentials( + {account.credential_name: "secret"}, + require_all=False, + accounts=(account,), + ) + service = catalog.ProviderCatalogService( + store=catalog.InMemoryProviderCatalogStore(), + accounts=(account,), + discover=lambda _account, _credential: [], + ) + + with pytest.raises(catalog.ProviderCatalogUnavailable, match="no usable provider model"): + service.refresh_all() + assert service.last_refresh_summary["provider_accounts"][account.provider_account_id] == { + "status": "failed", + "model_count": 0, + "error_code": "catalog_contains_no_models", + } + + +def test_bytez_string_output_streaming_and_passthrough_guard() -> None: + """Native Bytez text can be framed, while unsupported passthrough fails closed.""" + account = catalog.DEFAULT_PROVIDER_ACCOUNTS[2] + catalog.bootstrap_provider_credentials( + {account.credential_name: "bytez-secret"}, + require_all=False, + accounts=(account,), + ) + agent = ModelAgent( + "bytez_worker", + "owner/model", + account.base_url, + credential_key=account.credential_name, + provider_name="bytez", + ) + client = catalog.ProviderAwareModelClient( + bytez_request=lambda _agent, _messages, _credential: {"output": "native-answer"} + ) + assert "".join(client.stream_chat(agent, [{"role": "user", "content": "hello"}])) == "native-answer" + with pytest.raises(catalog.ProviderCatalogUnavailable, match="does not support passthrough"): + client.proxy_send(agent, "/responses", {}) + + +def test_non_bytez_stream_and_proxy_keep_existing_mock_behavior() -> None: + """Provider-aware delegation preserves the existing mock transport surfaces.""" + client = catalog.ProviderAwareModelClient() + agent = ModelAgent("general_agent", "mock-generalist", "mock://local") + chunks = list(client.stream_chat(agent, [{"role": "user", "content": "hello"}])) + assert "".join(chunks) + raw = client.proxy_send(agent, "/responses", {"input": "hello"}) + assert isinstance(raw, dict) + + +def test_agent_tags_cover_role_without_chat_capability() -> None: + """Reasoning-only models receive role tags without inheriting chat tags.""" + account = catalog.DEFAULT_PROVIDER_ACCOUNTS[0] + store = catalog.InMemoryProviderCatalogStore() + store.replace_catalog( + account, + [ + catalog.DiscoveredModel( + model_name="reasoning-only", + display_name="Reasoning Only", + capabilities=("reasoning",), + modalities=("text",), + context_window=128_000, + input_price_usd_per_million=1.0, + output_price_usd_per_million=1.0, + ) + ], + ) + + agent = catalog.ProviderCatalogService(store=store, accounts=(account,)).candidate_agents()[0] + + assert {"planning", "research", "verification"}.issubset(agent.tags) + assert "writing" not in agent.tags + + +def test_scalar_capability_and_extreme_context_helpers() -> None: + """Scalar provider metadata and oversized integer values are bounded.""" + model = catalog.normalize_models_document( + { + "data": [ + { + "id": "custom-model", + "capabilities": "SPECIAL", + "context_length": "10000000001", + "pricing": {"prompt": "not-a-number"}, + } + ] + } + )[0] + assert model.capabilities == ("chat", "special") + assert model.context_window is None + assert model.input_price_usd_per_million is None diff --git a/tests/test_provider_catalog_edge_cases.py b/tests/test_provider_catalog_edge_cases.py new file mode 100644 index 000000000..bc996769d --- /dev/null +++ b/tests/test_provider_catalog_edge_cases.py @@ -0,0 +1,48 @@ +"""Terminal retry and metadata edge cases for the provider catalog.""" + +from __future__ import annotations + +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.provider_catalog import ( # noqa: E402 + DEFAULT_PROVIDER_ACCOUNTS, + CatalogHttpError, + ProviderCatalogHttpClient, + normalize_models_document, +) + + +def test_terminal_transient_error_is_not_slept_or_retried() -> None: + """A one-attempt policy surfaces its stable transient code immediately.""" + sleeps: list[float] = [] + client = ProviderCatalogHttpClient(max_attempts=1, sleep=sleeps.append) + client._request_json = lambda _account, _credential: (_ for _ in ()).throw( # type: ignore[method-assign] + CatalogHttpError("catalog_http_503", transient=True) + ) + with pytest.raises(CatalogHttpError, match="catalog_http_503"): + client.discover(DEFAULT_PROVIDER_ACCOUNTS[0], "credential") + assert sleeps == [] + + +def test_boolean_context_and_empty_display_name_are_bounded() -> None: + """Boolean context metadata is rejected and empty display names fall back to ids.""" + model = normalize_models_document( + { + "data": [ + { + "id": "fallback-model", + "name": " ", + "context_length": True, + "pricing": {"prompt": False}, + } + ] + } + )[0] + assert model.display_name == "fallback-model" + assert model.context_window is None + assert model.input_price_usd_per_million is None diff --git a/tests/test_provider_catalog_output_shapes.py b/tests/test_provider_catalog_output_shapes.py new file mode 100644 index 000000000..a74c394a3 --- /dev/null +++ b/tests/test_provider_catalog_output_shapes.py @@ -0,0 +1,73 @@ +"""Native provider output variants and safe summary contracts.""" + +from __future__ import annotations + +import json +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + set_backend, +) +from contextual_orchestrator.orchestrator import ModelAgent # noqa: E402 +from contextual_orchestrator.provider_catalog import ( # noqa: E402 + DEFAULT_PROVIDER_ACCOUNTS, + ProviderAwareModelClient, + _safe_cli_summary, + bootstrap_provider_credentials, +) + + +def test_bytez_text_mapping_is_accepted_without_usage_fabrication() -> None: + """A native Bytez text field is accepted while usage remains explicitly absent.""" + set_backend(InMemoryCredentialBackend()) + try: + account = DEFAULT_PROVIDER_ACCOUNTS[2] + bootstrap_provider_credentials( + {account.credential_name: "secret-value"}, + require_all=False, + accounts=(account,), + ) + agent = ModelAgent( + "bytez_worker", + "owner/model", + account.base_url, + credential_key=account.credential_name, + provider_name="bytez", + ) + client = ProviderAwareModelClient( + bytez_request=lambda _agent, _messages, _credential: { + "output": {"text": "text-answer"} + } + ) + assert client.chat(agent, [{"role": "user", "content": "hello"}]) == "text-answer" + assert client.take_usage() is None + finally: + set_backend(None) + + +def test_safe_bootstrap_summary_exposes_names_and_counts_only() -> None: + """The CI summary is stable, JSON-serializable, and contains no unknown input fields.""" + summary = _safe_cli_summary( + { + "registered_credentials": ["OPENAI_API_KEY"], + "missing_credentials": ["BYTEZ_API_KEY"], + "secret_value": "must-not-copy", + }, + { + "candidate_model_count": 4, + "provider_accounts": {"openai_primary": {"status": "refreshed"}}, + "provider_body": "must-not-copy", + }, + ) + assert summary == { + "registered_credentials": ["OPENAI_API_KEY"], + "missing_credentials": ["BYTEZ_API_KEY"], + "candidate_model_count": 4, + "provider_accounts": {"openai_primary": {"status": "refreshed"}}, + "measurement_status": "provider_catalog_bootstrap", + } + assert "must-not-copy" not in json.dumps(summary) diff --git a/tests/test_provider_credential_revocation.py b/tests/test_provider_credential_revocation.py new file mode 100644 index 000000000..b779dddce --- /dev/null +++ b/tests/test_provider_credential_revocation.py @@ -0,0 +1,99 @@ +"""Provider egress must fail closed when a KV credential disappears after validation.""" + +from __future__ import annotations + +import http.client +import socket +from unittest import mock + +import pytest + +from contextual_orchestrator import ModelAgent +from contextual_orchestrator.credentials import ( + InMemoryCredentialBackend, + NotConfigured, + set_backend, +) +from contextual_orchestrator.orchestrator import ModelClient +from contextual_orchestrator.provider_transport import _PinnedHTTPSConnection + + +def _validated_provider() -> tuple[ModelClient, ModelAgent]: + """Return a client with one public-address pin established under a live credential.""" + backend = InMemoryCredentialBackend() + backend.set("MODEL_KEY", "sk-live-before-revocation") + set_backend(backend) + client = ModelClient() + agent = ModelAgent( + "remote_agent", + "gpt-example", + "https://provider.example/v1", + "MODEL_KEY", + ) + resolved = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 443))] + with mock.patch( + "contextual_orchestrator.orchestrator.socket.getaddrinfo", + return_value=resolved, + ): + client._validate_provider(agent) + return client, agent + + +def test_every_provider_egress_path_blocks_revoked_credential_before_socket() -> None: + """Revocation after DNS validation must stop every HTTPS request before socket egress.""" + client, agent = _validated_provider() + try: + set_backend(InMemoryCredentialBackend()) + operations = ( + lambda: client._send(agent, {"model": agent.model}), + lambda: list(client._stream_send(agent, {"model": agent.model, "stream": True})), + lambda: client._send_raw(agent, "responses", {"model": agent.model}), + lambda: client._batch_upload(agent, b"{}\n"), + lambda: client._batch_json(agent, "GET", "/batches/batch_1"), + lambda: client._batch_raw(agent, "/files/file_1/content"), + ) + for operation in operations: + with mock.patch( + "contextual_orchestrator.provider_transport.socket.create_connection", + side_effect=AssertionError("socket egress attempted after credential revocation"), + ): + with pytest.raises(NotConfigured, match="Bearer credential"): + operation() + finally: + set_backend(None) + + +def test_pinned_connection_rejects_missing_authorization_before_super_request() -> None: + """A direct pinned request without provider authorization also fails closed.""" + connection = _PinnedHTTPSConnection( + "provider.example", + "8.8.8.8", + 443, + 1.0, + mock.Mock(), + ) + with mock.patch.object(http.client.HTTPSConnection, "request") as base_request: + with pytest.raises(NotConfigured, match="Bearer credential"): + connection.request("POST", "/v1/chat/completions", headers={}) + base_request.assert_not_called() + + +def test_pinned_connection_accepts_nonempty_bearer_for_normal_dispatch() -> None: + """A current non-empty Bearer value reaches the standard HTTPS request machinery.""" + connection = _PinnedHTTPSConnection( + "provider.example", + "8.8.8.8", + 443, + 1.0, + mock.Mock(), + ) + headers = {"Authorization": "Bearer sk-current-value"} + with mock.patch.object(http.client.HTTPSConnection, "request") as base_request: + connection.request("POST", "/v1/chat/completions", headers=headers) + base_request.assert_called_once_with( + "POST", + "/v1/chat/completions", + body=None, + headers=headers, + encode_chunked=False, + ) diff --git a/tests/test_provider_json_boundary.py b/tests/test_provider_json_boundary.py new file mode 100644 index 000000000..2caf67bb0 --- /dev/null +++ b/tests/test_provider_json_boundary.py @@ -0,0 +1,304 @@ +"""Regression coverage for the provider JSON trust boundary. + +Provider-controlled structured responses are bounded before parsing, decoded as +strict UTF-8 JSON objects, and converted to stable errors that do not retain the +untrusted document in an exception cause. Validated HTTPS connections carry the +request path into the response wrapper so existing model-client call sites gain +the boundary without import-time mutation or duplicated parsing policy. Batch +output file content remains strict JSON Lines rather than a single JSON object. +""" + +from __future__ import annotations + +import http.client +import json +import ssl +from unittest import mock + +import pytest + +import contextual_orchestrator.provider_transport as provider_transport +from contextual_orchestrator.orchestrator import ModelAgent, ModelClient +from contextual_orchestrator.provider_transport import ( + _PinnedHTTPSConnection, + _ProviderHTTPResponse, + _is_batch_output_content_path, +) + + +class _ByteResponse: + """Small byte response with observable cleanup for decoder regressions.""" + + def __init__(self, payload: bytes) -> None: + """Store one provider-controlled body and initialize cleanup evidence.""" + self._payload = payload + self.closed = False + + def read(self, amount: int | None = None) -> bytes: + """Return at most the requested bytes, matching ``HTTPResponse.read``.""" + if amount is None or amount < 0: + amount = len(self._payload) + chunk = self._payload[:amount] + self._payload = self._payload[amount:] + return chunk + + def close(self) -> None: + """Record deterministic response cleanup.""" + self.closed = True + + +def _provider_agent() -> ModelAgent: + """Return one two-word-ID HTTPS agent suitable for transport-unit seams.""" + return ModelAgent( + id="provider_agent", + model="provider-model", + base_url="https://provider.example", + credential_key="NVIDIA_NIM_API_KEY", + ) + + +def _path_aware_wrapper(payload: bytes, path: str) -> tuple[_ProviderHTTPResponse, _ByteResponse, mock.Mock]: + """Return one response wrapper carrying the validated provider request path.""" + response = _ByteResponse(payload) + connection = mock.Mock() + connection._provider_request_path = path + return _ProviderHTTPResponse(response, connection, max_bytes=4096), response, connection + + +@pytest.mark.parametrize( + ("payload", "private_marker"), + [ + (b'{"secret":"private-json-value",', "private-json-value"), + (b'{"secret":"\xffprivate-utf8-value"}', "private-utf8-value"), + ], +) +def test_malformed_provider_json_is_redacted_without_exception_cause( + payload: bytes, + private_marker: str, +) -> None: + """Malformed syntax or UTF-8 cannot retain provider text in error evidence.""" + response = _ByteResponse(payload) + connection = mock.Mock() + wrapper = _ProviderHTTPResponse(response, connection, max_bytes=512) + + with pytest.raises(RuntimeError, match="provider JSON response is malformed") as error: + with wrapper: + wrapper.read_json_object() + + assert error.value.__cause__ is None + assert private_marker not in str(error.value) + assert private_marker not in repr(error.value) + assert response.closed is True + connection.close.assert_called_once_with() + + +@pytest.mark.parametrize("constant", [b"NaN", b"Infinity", b"-Infinity"]) +def test_provider_json_rejects_non_finite_number_extensions(constant: bytes) -> None: + """Python-specific non-finite extensions cannot cross the RFC 8259 boundary.""" + response = _ByteResponse(b'{"value":' + constant + b"}") + wrapper = _ProviderHTTPResponse(response, mock.Mock(), max_bytes=512) + + with pytest.raises(RuntimeError, match="provider JSON response is malformed"): + wrapper.read_json_object() + + +def test_provider_json_rejects_duplicate_object_member_names() -> None: + """Duplicate member names fail closed instead of inheriting last-value wins.""" + response = _ByteResponse(b'{"choice":1,"choice":2}') + wrapper = _ProviderHTTPResponse(response, mock.Mock(), max_bytes=512) + + with pytest.raises(RuntimeError, match="provider JSON response is malformed"): + wrapper.read_json_object() + + +def test_provider_json_requires_top_level_object() -> None: + """A syntactically valid scalar or array is not an OpenAI-compatible object.""" + response = _ByteResponse(b"[]") + wrapper = _ProviderHTTPResponse(response, mock.Mock(), max_bytes=512) + + with pytest.raises(RuntimeError, match="provider JSON response must be an object"): + wrapper.read_json_object() + + +def test_provider_json_accepts_valid_utf8_object() -> None: + """Valid UTF-8 JSON objects preserve Unicode content after bounded parsing.""" + response = _ByteResponse('{"message":"안녕하세요","count":2}'.encode("utf-8")) + wrapper = _ProviderHTTPResponse(response, mock.Mock(), max_bytes=512) + + assert wrapper.read_json_object() == {"message": "안녕하세요", "count": 2} + + +def test_provider_json_canonicalization_redacts_unserializable_values( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A decoded value that cannot be canonicalized fails without leaking input.""" + private_marker = b"private-provider-marker" + monkeypatch.setattr( + provider_transport, + "_decode_provider_json_object", + lambda _payload: {"value": object()}, + ) + + with pytest.raises(RuntimeError, match="provider JSON response is malformed") as error: + provider_transport._encode_provider_json_object(private_marker) + + assert error.value.__cause__ is None + assert private_marker.decode() not in str(error.value) + assert private_marker.decode() not in repr(error.value) + + +def test_pinned_https_connection_records_response_contract_path() -> None: + """The exact validated request target accompanies its later response wrapper.""" + connection = _PinnedHTTPSConnection( + "provider.example", + "203.0.113.10", + 443, + 1.0, + ssl.create_default_context(), + ) + + with mock.patch.object(http.client.HTTPSConnection, "request") as parent_request: + connection.request( + "POST", + "/v1/chat/completions?trace=one", + headers={"Authorization": "Bearer reviewed-secret"}, + ) + + assert connection._provider_request_path == "/v1/chat/completions?trace=one" + parent_request.assert_called_once() + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + ("/v1/files/file-output/content", True), + ("/files/file-output/content?download=1", True), + ("/v1/files", False), + ("/v1/chat/completions", False), + ("/v1/files/file-output/metadata", False), + ], +) +def test_batch_output_path_classification_is_narrow(path: str, expected: bool) -> None: + """Only the exact file-content suffix receives JSON Lines semantics.""" + assert _is_batch_output_content_path(path) is expected + + +def test_model_client_json_object_paths_fail_closed_through_transport() -> None: + """Every structured provider path redacts malformed response documents.""" + client = ModelClient() + agent = _provider_agent() + malformed = b'{"secret":"private-provider-document",' + + wrappers = [ + _path_aware_wrapper(malformed, "/v1/chat/completions")[0], + _path_aware_wrapper(malformed, "/v1/responses")[0], + _path_aware_wrapper(malformed, "/v1/files")[0], + _path_aware_wrapper(malformed, "/v1/batches/batch-id")[0], + ] + calls = [ + lambda: client._send(agent, {"model": agent.model}), + lambda: client._send_raw(agent, "responses", {"model": agent.model}), + lambda: client._batch_upload(agent, b"{}\n"), + lambda: client._batch_json(agent, "GET", "/batches/batch-id"), + ] + + for wrapper, call in zip(wrappers, calls, strict=True): + client._open_provider = mock.Mock(return_value=wrapper) # type: ignore[method-assign] + with pytest.raises(RuntimeError, match="provider JSON response is malformed") as error: + call() + assert error.value.__cause__ is None + assert "private-provider-document" not in str(error.value) + + +def test_model_client_json_object_paths_preserve_valid_results() -> None: + """The shared strict boundary preserves existing structured response semantics.""" + client = ModelClient() + agent = _provider_agent() + + wrapper, _, _ = _path_aware_wrapper( + json.dumps( + { + "choices": [{"message": {"content": "ok"}}], + "usage": {"total_tokens": 1}, + } + ).encode("utf-8"), + "/v1/chat/completions", + ) + client._open_provider = mock.Mock(return_value=wrapper) # type: ignore[method-assign] + assert client._send(agent, {"model": agent.model}) == "ok" + + wrapper, _, _ = _path_aware_wrapper(b'{"id":"response-id"}', "/v1/responses") + client._open_provider = mock.Mock(return_value=wrapper) # type: ignore[method-assign] + assert client._send_raw(agent, "responses", {"model": agent.model}) == { + "id": "response-id" + } + + wrapper, _, _ = _path_aware_wrapper(b'{"id":"file-id"}', "/v1/files") + client._open_provider = mock.Mock(return_value=wrapper) # type: ignore[method-assign] + assert client._batch_upload(agent, b"{}\n") == "file-id" + + wrapper, _, _ = _path_aware_wrapper(b'{"id":"batch-id"}', "/v1/batches/batch-id") + client._open_provider = mock.Mock(return_value=wrapper) # type: ignore[method-assign] + assert client._batch_json(agent, "GET", "/batches/batch-id") == {"id": "batch-id"} + + +def test_batch_output_path_preserves_strict_json_lines_and_blank_separators() -> None: + """Batch output remains line-addressable while harmless blank rows are ignored.""" + wrapper, _, _ = _path_aware_wrapper( + b'{"custom_id":"first","value":1}\n\n{"custom_id":"second","value":2}\n', + "/v1/files/file-output/content", + ) + + rows = [json.loads(line) for line in wrapper.read().decode("utf-8").splitlines()] + assert rows == [ + {"custom_id": "first", "value": 1}, + {"custom_id": "second", "value": 2}, + ] + + +@pytest.mark.parametrize( + "payload", + [ + b'{"custom_id":"private-row","value":1,"value":2}\n', + b'{"custom_id":"\xffprivate-row"}\n', + b"[]\n", + b"\n \t\n", + ], +) +def test_batch_output_json_lines_reject_malformed_or_non_object_rows(payload: bytes) -> None: + """Invalid JSONL fails before later row parsing can retain provider content.""" + wrapper, response, connection = _path_aware_wrapper( + payload, + "/v1/files/file-output/content?download=1", + ) + + with pytest.raises(RuntimeError, match="provider JSON Lines response is malformed") as error: + with wrapper: + wrapper.read() + + assert error.value.__cause__ is None + assert "private-row" not in str(error.value) + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_path_aware_partial_read_retains_bounded_byte_semantics() -> None: + """Explicit partial reads remain byte-oriented and are never parsed prematurely.""" + wrapper, _, _ = _path_aware_wrapper(b'{"value":1}', "/v1/chat/completions") + assert wrapper.read(2) == b'{"' + + +def test_path_aware_negative_read_normalizes_complete_document() -> None: + """A negative read amount means full-document validation, matching HTTPResponse.""" + wrapper, _, _ = _path_aware_wrapper(b'{ "value" : 1 }', "/v1/chat/completions") + assert wrapper.read(-1) == b'{"value":1}' + + +def test_empty_request_path_preserves_byte_oriented_test_seam() -> None: + """An uncaptured test seam never gains provider-document authority accidentally.""" + response = _ByteResponse(b"not-json") + connection = mock.Mock() + connection._provider_request_path = "" + wrapper = _ProviderHTTPResponse(response, connection, max_bytes=512) + + assert wrapper.read() == b"not-json" diff --git a/tests/test_provider_json_finite_number_boundary.py b/tests/test_provider_json_finite_number_boundary.py new file mode 100644 index 000000000..e04c10115 --- /dev/null +++ b/tests/test_provider_json_finite_number_boundary.py @@ -0,0 +1,59 @@ +"""Regression coverage for finite provider JSON numeric decoding. + +RFC 8259 permits exponent notation, while Python's binary-float decoder can +materialize a syntactically valid but extreme exponent such as ``1e999`` as an +infinite runtime value. Provider-controlled JSON must never turn that parser +artifact into orchestration state, including the parser shared by streaming +server-sent events. +""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from contextual_orchestrator.provider_transport import _ProviderHTTPResponse + + +class _ByteResponse: + """Provide one deterministic provider body to the bounded response wrapper.""" + + def __init__(self, payload: bytes) -> None: + """Store the provider-controlled bytes for one complete read.""" + self._payload = payload + + def read(self, amount: int | None = None) -> bytes: + """Return at most ``amount`` bytes using ``HTTPResponse.read`` semantics.""" + if amount is None or amount < 0: + amount = len(self._payload) + chunk = self._payload[:amount] + self._payload = self._payload[amount:] + return chunk + + def close(self) -> None: + """Match the response cleanup protocol used by the wrapper.""" + + +@pytest.mark.parametrize("number", [b"1e999", b"-1e999"]) +def test_provider_json_rejects_float_overflow_to_infinity(number: bytes) -> None: + """Valid JSON exponents that overflow Python floats must fail closed.""" + wrapper = _ProviderHTTPResponse( + _ByteResponse(b'{"value":' + number + b"}"), + mock.Mock(), + max_bytes=512, + ) + + with pytest.raises(RuntimeError, match="provider JSON response is malformed"): + wrapper.read_json_object() + + +def test_provider_json_preserves_finite_exponent_numbers() -> None: + """Ordinary finite exponent notation remains accepted after the hardening.""" + wrapper = _ProviderHTTPResponse( + _ByteResponse(b'{"value":1.25e2}'), + mock.Mock(), + max_bytes=512, + ) + + assert wrapper.read_json_object() == {"value": 125.0} diff --git a/tests/test_provider_reliability.py b/tests/test_provider_reliability.py index dd5ead98c..6d30ec942 100644 --- a/tests/test_provider_reliability.py +++ b/tests/test_provider_reliability.py @@ -11,6 +11,8 @@ import sys import urllib.error +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 @@ -61,6 +63,12 @@ def _send(self, agent: ModelAgent, payload: dict) -> str: # type: ignore[overri assert all(0.0 <= d <= client.retry_backoff_cap for d in delays) +def test_negative_retry_budget_is_rejected() -> None: + """A retry budget cannot disable the provider's mandatory first attempt.""" + with pytest.raises(ValueError, match="max_retries must be at least zero"): + ModelClient(max_retries=-1) + + def test_permanent_error_is_not_retried() -> None: class BadRequestClient(ModelClient): def __init__(self) -> None: diff --git a/tests/test_provider_response_bounds.py b/tests/test_provider_response_bounds.py new file mode 100644 index 000000000..e5b053a49 --- /dev/null +++ b/tests/test_provider_response_bounds.py @@ -0,0 +1,333 @@ +"""Regression tests for bounded provider-response consumption. + +These tests keep untrusted provider bodies from becoming an unbounded memory or +stream-processing surface. The production wrapper must enforce one cumulative +byte budget across ordinary reads and server-sent-event iteration and must close +both response and connection resources when the budget is exceeded. +""" + +from __future__ import annotations + +import http.client +from unittest import mock + +import pytest + +from contextual_orchestrator import provider_transport +from contextual_orchestrator.provider_transport import _ProviderHTTPResponse + + +class _ReadableResponse: + """Small byte-stream double with observable bounded read calls.""" + + def __init__(self, payload: bytes) -> None: + """Store the payload and initialize read and cleanup evidence.""" + self._payload = payload + self._offset = 0 + self.read_sizes: list[int | None] = [] + self.closed = False + + def read(self, amount: int | None = None) -> bytes: + """Return at most ``amount`` bytes while recording the requested bound.""" + self.read_sizes.append(amount) + if amount is None or amount < 0: + amount = len(self._payload) - self._offset + start = self._offset + self._offset = min(len(self._payload), start + amount) + return self._payload[start : self._offset] + + def close(self) -> None: + """Record deterministic response cleanup.""" + self.closed = True + + def __iter__(self): + """Yield the unread payload as one line for non-HTTP test-double fallback.""" + if self._offset >= len(self._payload): + return iter(()) + payload = self._payload[self._offset :] + self._offset = len(self._payload) + return iter((payload,)) + + +class _BoundedHTTPResponse(http.client.HTTPResponse): + """HTTPResponse-shaped SSE double whose readline calls expose their byte bound.""" + + def __init__(self, lines: list[bytes]) -> None: + """Initialize line data without opening a real socket.""" + self._lines = list(lines) + self.readline_limits: list[int] = [] + self._closed_record = False + + @property + def closed(self) -> bool: + """Expose cleanup state without relying on an uninitialized IOBase socket.""" + return self._closed_record + + def getheader(self, name: str, default: str | None = None) -> str | None: + """Expose the SSE media type while leaving framing headers absent.""" + if name.lower() == "content-type": + return "text/event-stream; charset=utf-8" + return default + + def readline(self, limit: int = -1) -> bytes: + """Return one line, respecting the caller's requested maximum size.""" + self.readline_limits.append(limit) + if not self._lines: + return b"" + line = self._lines.pop(0) + if limit >= 0 and len(line) > limit: + self._lines.insert(0, line[limit:]) + return line[:limit] + return line + + def close(self) -> None: + """Record deterministic response cleanup.""" + self._closed_record = True + + +class _HeaderHTTPResponse(_BoundedHTTPResponse): + """HTTP response double exposing provider-controlled framing headers.""" + + def __init__(self, headers: dict[str, str]) -> None: + """Store case-insensitive headers without reading a response body.""" + super().__init__([]) + self._headers = {name.lower(): value for name, value in headers.items()} + + def getheader(self, name: str, default: str | None = None) -> str | None: + """Return one response header using HTTP's case-insensitive field names.""" + return self._headers.get(name.lower(), default) + + +class _FailingHeaderHTTPResponse(_BoundedHTTPResponse): + """HTTP response double whose provider-header lookup fails with private detail.""" + + def getheader(self, _name: str, default: str | None = None) -> str | None: + """Raise one provider-controlled metadata failure before returning a value.""" + del default + raise OSError("private upstream header detail") + + +class _FailingStreamHeaderHTTPResponse(_BoundedHTTPResponse): + """HTTP response double that fails only while reading its stream media type.""" + + def getheader(self, name: str, default: str | None = None) -> str | None: + """Allow framing validation, then fail on the later Content-Type lookup.""" + if name.lower() == "content-type": + raise OSError("private stream header detail") + return default + + +def test_default_provider_response_budget_is_eight_mibibytes() -> None: + """The reviewed default keeps every provider response below eight MiB.""" + assert provider_transport.PROVIDER_RESPONSE_MAX_BYTES == 8 * 1024 * 1024 + + +@pytest.mark.parametrize("invalid_limit", [0, -1, True, 1.5]) +def test_provider_response_rejects_invalid_byte_budget(invalid_limit: object) -> None: + """A non-positive, boolean, or non-integer byte budget fails closed.""" + with pytest.raises(ValueError, match="positive integer"): + _ProviderHTTPResponse(_ReadableResponse(b""), mock.Mock(), max_bytes=invalid_limit) + + +@pytest.mark.parametrize("declared_length", ["5", "100"]) +def test_oversized_declared_length_fails_before_body_read_and_closes_resources( + declared_length: str, +) -> None: + """An over-limit Content-Length is rejected before provider bytes are consumed.""" + response = _HeaderHTTPResponse({"Content-Length": declared_length}) + connection = mock.Mock() + + with pytest.raises(RuntimeError, match="response byte limit"): + _ProviderHTTPResponse(response, connection, max_bytes=4) + + assert response.readline_limits == [] + assert response.closed is True + connection.close.assert_called_once_with() + + +@pytest.mark.parametrize( + "declared_length", + [ + "", + "-1", + "+1", + "1.0", + "1, 2", + "4,,4", + "١", + "\u00a04", + "4\u00a0", + "\v4", + "4\f", + ], +) +def test_invalid_or_conflicting_declared_lengths_fail_closed( + declared_length: str, +) -> None: + """Malformed, non-ASCII, or conflicting Content-Length evidence is rejected.""" + response = _HeaderHTTPResponse({"Content-Length": declared_length}) + connection = mock.Mock() + + with pytest.raises(RuntimeError, match="content length"): + _ProviderHTTPResponse(response, connection, max_bytes=4) + + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_equal_duplicate_declared_lengths_are_normalized_without_body_reads() -> None: + """RFC-compatible repeated equal decimal lengths remain valid at the limit.""" + response = _HeaderHTTPResponse({"Content-Length": "\t0004\t, 4 "}) + connection = mock.Mock() + + wrapper = _ProviderHTTPResponse(response, connection, max_bytes=4) + assert response.readline_limits == [] + wrapper.close() + + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_declared_length_below_budget_is_accepted_without_body_reads() -> None: + """A valid shorter declared body remains subject to later cumulative reads.""" + response = _HeaderHTTPResponse({"Content-Length": "4"}) + connection = mock.Mock() + + wrapper = _ProviderHTTPResponse(response, connection, max_bytes=40) + assert response.readline_limits == [] + wrapper.close() + + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_content_length_with_transfer_encoding_is_rejected_as_ambiguous() -> None: + """Conflicting framing metadata cannot select a less-bounded body path.""" + response = _HeaderHTTPResponse( + {"Content-Length": "4", "Transfer-Encoding": "chunked"} + ) + connection = mock.Mock() + + with pytest.raises(RuntimeError, match="framing is ambiguous"): + _ProviderHTTPResponse(response, connection, max_bytes=4) + + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_header_lookup_failure_is_redacted_and_closes_resources() -> None: + """Provider metadata failures do not expose private text or leak the socket.""" + response = _FailingHeaderHTTPResponse([]) + connection = mock.Mock() + + with pytest.raises(RuntimeError, match="headers could not be validated") as error: + _ProviderHTTPResponse(response, connection, max_bytes=4) + + assert "private upstream header detail" not in str(error.value) + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_unbounded_read_probes_one_byte_past_remaining_budget() -> None: + """A full read detects one-byte overflow without first buffering the whole body.""" + response = _ReadableResponse(b"abcde") + connection = mock.Mock() + wrapper = _ProviderHTTPResponse(response, connection, max_bytes=4) + + with pytest.raises(RuntimeError, match="response byte limit"): + with wrapper: + wrapper.read() + + assert response.read_sizes == [5] + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_exact_limit_read_succeeds_and_explicit_small_reads_are_preserved() -> None: + """Bodies at the limit succeed and caller-requested smaller reads stay bounded.""" + response = _ReadableResponse(b"abcd") + connection = mock.Mock() + wrapper = _ProviderHTTPResponse(response, connection, max_bytes=4) + + with wrapper: + assert wrapper.read(2) == b"ab" + assert wrapper.read(2) == b"cd" + assert wrapper.read() == b"" + + assert response.read_sizes == [2, 2, 1] + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_explicit_read_larger_than_remaining_budget_cannot_bypass_limit() -> None: + """A later oversized explicit read probes only remaining bytes plus one.""" + response = _ReadableResponse(b"abcde") + wrapper = _ProviderHTTPResponse(response, mock.Mock(), max_bytes=4) + + assert wrapper.read(2) == b"ab" + with pytest.raises(RuntimeError, match="response byte limit"): + wrapper.read(99) + assert response.read_sizes == [2, 3] + + +def test_negative_read_amount_is_treated_as_full_bounded_read() -> None: + """HTTPResponse's negative full-read convention remains subject to the cap.""" + response = _ReadableResponse(b"abcde") + wrapper = _ProviderHTTPResponse(response, mock.Mock(), max_bytes=4) + + with pytest.raises(RuntimeError, match="response byte limit"): + wrapper.read(-1) + assert response.read_sizes == [5] + + +def test_http_iteration_uses_bounded_readline_and_accepts_exact_budget() -> None: + """SSE iteration never asks the socket for more than remaining budget plus one.""" + response = _BoundedHTTPResponse([b":a\n", b"data: [DONE]\n"]) + connection = mock.Mock() + wrapper = _ProviderHTTPResponse(response, connection, max_bytes=16) + + with wrapper: + assert list(wrapper) == [b":a\n"] + + assert response.readline_limits == [17, 14] + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_http_iteration_rejects_cumulative_overflow_and_closes_resources() -> None: + """A streaming provider cannot exceed the cumulative cap across multiple lines.""" + response = _BoundedHTTPResponse([b":a\n", b":bc\n"]) + connection = mock.Mock() + wrapper = _ProviderHTTPResponse(response, connection, max_bytes=5) + + with pytest.raises(RuntimeError, match="response byte limit"): + with wrapper: + list(wrapper) + + assert response.readline_limits == [6, 3] + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_http_iteration_rejects_stream_content_type_lookup_failure() -> None: + """A provider Content-Type lookup failure is redacted and closes resources.""" + response = _FailingStreamHeaderHTTPResponse([b"data: [DONE]\n"]) + connection = mock.Mock() + wrapper = _ProviderHTTPResponse(response, connection, max_bytes=32) + + with pytest.raises(RuntimeError, match="content type could not be validated") as error: + with wrapper: + list(wrapper) + + assert "private stream header detail" not in str(error.value) + assert response.closed is True + connection.close.assert_called_once_with() + + +def test_non_http_response_iteration_fallback_is_still_cumulatively_bounded() -> None: + """Existing lightweight response doubles retain iteration with the same cap.""" + response = _ReadableResponse(b"abcde") + wrapper = _ProviderHTTPResponse(response, mock.Mock(), max_bytes=4) + + with pytest.raises(RuntimeError, match="response byte limit"): + list(wrapper) diff --git a/tests/test_provider_stream_utf8.py b/tests/test_provider_stream_utf8.py new file mode 100644 index 000000000..dbb679a86 --- /dev/null +++ b/tests/test_provider_stream_utf8.py @@ -0,0 +1,58 @@ +"""Regression coverage for fail-closed UTF-8 server-sent-event decoding.""" + +from __future__ import annotations + +import http.client +from unittest import mock + +import pytest + +from contextual_orchestrator.provider_transport import _ProviderHTTPResponse + + +class _InvalidUtf8HTTPResponse(http.client.HTTPResponse): + """HTTPResponse-shaped SSE double that emits one malformed UTF-8 line.""" + + def __init__(self) -> None: + """Initialize malformed stream bytes without opening a socket.""" + self._lines = [b"data: \xffprivate-upstream-detail\n"] + self._closed_record = False + + @property + def closed(self) -> bool: + """Expose deterministic cleanup state.""" + return self._closed_record + + def getheader(self, name: str, default: str | None = None) -> str | None: + """Expose only a valid SSE media type; framing headers remain absent.""" + if name.lower() == "content-type": + return "text/event-stream; charset=utf-8" + return default + + def readline(self, limit: int = -1) -> bytes: + """Return malformed provider bytes while respecting the read bound.""" + if not self._lines: + return b"" + line = self._lines.pop(0) + if limit >= 0: + return line[:limit] + return line + + def close(self) -> None: + """Record deterministic response cleanup.""" + self._closed_record = True + + +def test_invalid_utf8_sse_is_redacted_and_closes_resources() -> None: + """Malformed provider UTF-8 fails closed without exposing provider text.""" + response = _InvalidUtf8HTTPResponse() + connection = mock.Mock() + wrapper = _ProviderHTTPResponse(response, connection, max_bytes=64) + + with pytest.raises(RuntimeError, match="malformed provider stream event") as error: + with wrapper: + list(wrapper) + + assert "private-upstream-detail" not in str(error.value) + assert response.closed is True + connection.close.assert_called_once_with() diff --git a/tests/test_provider_transfer_encoding.py b/tests/test_provider_transfer_encoding.py new file mode 100644 index 000000000..79578f4d7 --- /dev/null +++ b/tests/test_provider_transfer_encoding.py @@ -0,0 +1,83 @@ +"""Regression tests for fail-closed provider Transfer-Encoding handling. + +The provider transport supports the HTTP/1.1 ``chunked`` transfer coding that +Python's ``http.client`` decodes. Other transfer-coding chains are valid in parts +of HTTP/1.1 but are outside this product's decoding contract, so they must be +rejected before application parsing rather than treated as an opaque body. +""" + +from __future__ import annotations + +import http.client +from unittest import mock + +import pytest + +from contextual_orchestrator.provider_transport import _ProviderHTTPResponse + + +class _TransferEncodingResponse(http.client.HTTPResponse): + """HTTPResponse-shaped double exposing only reviewed framing metadata.""" + + def __init__(self, transfer_encoding: str | None) -> None: + """Store one provider-controlled Transfer-Encoding field value.""" + self._transfer_encoding = transfer_encoding + self._closed_record = False + + @property + def closed(self) -> bool: + """Expose deterministic cleanup without an initialized socket file.""" + return self._closed_record + + def getheader(self, name: str, default: str | None = None) -> str | None: + """Return framing headers using HTTP's case-insensitive field names.""" + lowered = name.lower() + if lowered == "transfer-encoding": + return self._transfer_encoding + if lowered == "content-length": + return None + return default + + def close(self) -> None: + """Record deterministic response cleanup.""" + self._closed_record = True + + +@pytest.mark.parametrize( + "transfer_encoding", + [ + "gzip", + "gzip, chunked", + "chunked, chunked", + "chunked;foo=bar", + "identity", + "", + ], +) +def test_unsupported_transfer_encoding_fails_closed_before_body_consumption( + transfer_encoding: str, +) -> None: + """Only the stdlib-decoded single chunked coding is accepted by the product.""" + response = _TransferEncodingResponse(transfer_encoding) + connection = mock.Mock() + + with pytest.raises(RuntimeError, match="transfer encoding is unsupported"): + _ProviderHTTPResponse(response, connection, max_bytes=32) + + assert response.closed is True + connection.close.assert_called_once_with() + + +@pytest.mark.parametrize("transfer_encoding", ["chunked", "Chunked", "CHUNKED"]) +def test_single_chunked_transfer_encoding_remains_supported( + transfer_encoding: str, +) -> None: + """HTTP/1.1 chunked framing remains compatible with bounded body reads.""" + response = _TransferEncodingResponse(transfer_encoding) + connection = mock.Mock() + + wrapper = _ProviderHTTPResponse(response, connection, max_bytes=32) + wrapper.close() + + assert response.closed is True + connection.close.assert_called_once_with() diff --git a/tests/test_repository_coverage_policy.py b/tests/test_repository_coverage_policy.py new file mode 100644 index 000000000..af551972d --- /dev/null +++ b/tests/test_repository_coverage_policy.py @@ -0,0 +1,54 @@ +"""Repository contracts for fail-closed production coverage and public docstrings.""" + +from pathlib import Path +import tomli as tomllib + + +ROOT_DIR = Path(__file__).resolve().parents[1] +PYPROJECT_PATH = ROOT_DIR / "pyproject.toml" +TESTS_WORKFLOW_PATH = ROOT_DIR / ".github" / "workflows" / "tests.yml" + + +def _pyproject() -> dict[str, object]: + """Return the parsed project configuration used by local and CI evidence.""" + return tomllib.loads(PYPROJECT_PATH.read_text(encoding="utf-8")) + + +def _named_step(workflow: str, name: str) -> str: + """Return one exact named workflow step without depending on a YAML package.""" + marker = f" - name: {name}\n" + start = workflow.index(marker) + try: + end = workflow.index("\n - name:", start + len(marker)) + except ValueError: + end = len(workflow) + return workflow[start:end] + + +def test_production_coverage_policy_is_branch_complete_and_has_no_omissions() -> None: + """Require every production module to participate in the 100% branch gate.""" + config = _pyproject() + coverage = config["tool"]["coverage"] + + assert coverage["run"].get("omit", []) == [] + assert coverage["run"]["branch"] is True + assert coverage["report"]["fail_under"] == 100 + + +def test_public_docstring_policy_requires_complete_evidence() -> None: + """Require the package public-docstring threshold to be exactly 100 percent.""" + config = _pyproject() + + assert config["tool"]["interrogate"]["fail-under"] == 100 + + +def test_exact_head_tests_workflow_enforces_coverage_and_docstrings() -> None: + """Keep repository-local exact-head CI fail-closed on both evidence classes.""" + workflow = TESTS_WORKFLOW_PATH.read_text(encoding="utf-8") + install_step = _named_step(workflow, "Install test dependencies") + test_step = _named_step(workflow, "Run full test suite") + + assert "requirements-opencode-review-ci.txt" in install_step + assert "python -m coverage run --branch -m pytest -q" in test_step + assert "python -m coverage report --fail-under=100" in test_step + assert "interrogate --fail-under 100 contextual_orchestrator" in test_step diff --git a/tests/test_security_hardening.py b/tests/test_security_hardening.py index 67134ea6b..715995216 100644 --- a/tests/test_security_hardening.py +++ b/tests/test_security_hardening.py @@ -2,11 +2,13 @@ import json import os +import socket import threading import urllib.error import urllib.request from pathlib import Path import sys +from unittest import mock sys.path.insert(0, str(Path(__file__).resolve().parents[1])) @@ -311,6 +313,60 @@ def test_provider_transport_rejects_protocol_relative_batch_paths() -> None: raise AssertionError("protocol-relative provider path should fail before urllib opens it") +def test_external_provider_rejects_non_global_resolved_addresses() -> None: + # The egress guard must reject ANY non-globally-routable resolved address, not + # only the RFC1918/loopback/link-local/multicast/reserved flag set. RFC 6598 + # shared address space (100.64.0.0/10, carrier-grade NAT and commonly used for + # cloud-internal services) is non-public yet carries NONE of those flags, so a + # host that resolves into it must still be blocked or it becomes an SSRF hole to + # internal targets. getaddrinfo is stubbed so the check is deterministic offline. + client = ModelClient() + backend = InMemoryCredentialBackend() + backend.set("MODEL_KEY", "sk-shared-space") + set_backend(backend) + shared_space_agent = ModelAgent( + "shared_space_agent", "gpt-example", "https://provider.example/v1", "MODEL_KEY" + ) + resolved = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("100.64.0.1", 443))] + try: + with mock.patch( + "contextual_orchestrator.orchestrator.socket.getaddrinfo", + return_value=resolved, + ): + try: + client._validate_provider(shared_space_agent) + except RuntimeError as exc: + assert "non-public address" in str(exc) + else: + raise AssertionError( + "provider resolving to RFC 6598 shared address space " + "(100.64.0.0/10) must be rejected by the egress guard" + ) + finally: + set_backend(None) + + +def test_external_provider_allows_public_resolved_address() -> None: + # The tightened guard must not over-block: a genuinely public, globally + # routable resolved address still passes validation. + client = ModelClient() + backend = InMemoryCredentialBackend() + backend.set("MODEL_KEY", "sk-public") + set_backend(backend) + public_agent = ModelAgent( + "public_agent", "gpt-example", "https://provider.example/v1", "MODEL_KEY" + ) + resolved = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 443))] + try: + with mock.patch( + "contextual_orchestrator.orchestrator.socket.getaddrinfo", + return_value=resolved, + ): + client._validate_provider(public_agent) # must not raise + finally: + set_backend(None) + + def test_redact_value_preserves_non_string_scalars() -> None: assert redact_value(7) == 7 @@ -330,5 +386,7 @@ def test_redact_value_preserves_non_string_scalars() -> None: test_external_provider_rejects_insecure_or_unlisted_hosts() test_provider_transport_rejects_local_url_schemes_before_urllib() test_provider_transport_rejects_protocol_relative_batch_paths() + test_external_provider_rejects_non_global_resolved_addresses() + test_external_provider_allows_public_resolved_address() test_redact_value_preserves_non_string_scalars() print("ok") diff --git a/tests/test_server_coverage.py b/tests/test_server_coverage.py new file mode 100644 index 000000000..37412822f --- /dev/null +++ b/tests/test_server_coverage.py @@ -0,0 +1,574 @@ +"""Behavioural coverage for HTTP validation, admin surfaces, and operator CRUD.""" + +from __future__ import annotations + +import contextlib +import json +from pathlib import Path +import sys +import threading +import time +import urllib.error +import urllib.request + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +import contextual_orchestrator.server as server_module # noqa: E402 +from contextual_orchestrator.server import ( # noqa: E402 + RequestError, + SecurityConfig, + _coerce_json, + _embeddings_attribution, + _response_payload, + _validate_attribution, + _validate_batch_requests, + _validate_embeddings_inputs, + _validate_messages, + _validate_mode, + _validate_routing, + build_server, +) + +TOKEN = "coverage-token" + + +def _build() -> TaskOrchestrator: + return TaskOrchestrator( + [ + ModelAgent("general_agent", "mock-generalist", tags=("reasoning", "writing", "planning", "research")), + ModelAgent("review_agent", "mock-reviewer", tags=("verification", "security", "review")), + ] + ) + + +@contextlib.contextmanager +def _running_server(orchestrator: TaskOrchestrator, **security_kwargs): + security = SecurityConfig(auth_token=TOKEN, rate_limit_requests=1000, **security_kwargs) + server = build_server(orchestrator, port=0, security=security) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() + + +def _request( + base_url: str, + method: str, + path: str, + payload: object | None = None, + *, + token: str | None = TOKEN, + content_type: str = "application/json", + raw_body: bytes | None = None, + extra_headers: dict[str, str] | None = None, +) -> tuple[int, object, dict[str, str]]: + headers = {"connection": "close"} + if token is not None: + headers["authorization"] = f"Bearer {token}" + if payload is not None or raw_body is not None: + headers["content-type"] = content_type + if extra_headers: + headers.update(extra_headers) + data = raw_body if raw_body is not None else (json.dumps(payload).encode("utf-8") if payload is not None else None) + request = urllib.request.Request(base_url + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=10) as response: + body = response.read() + parsed: object + if response.headers.get("content-type", "").startswith("application/json"): + parsed = json.loads(body.decode("utf-8")) + else: + parsed = body.decode("utf-8") + return response.status, parsed, dict(response.headers.items()) + except urllib.error.HTTPError as exc: + body = exc.read() + parsed = json.loads(body.decode("utf-8")) if body else {} + return exc.code, parsed, dict(exc.headers.items()) + + +def test_security_and_validation_branches_fail_closed(monkeypatch: pytest.MonkeyPatch) -> None: + """Validation helpers reject malformed security, request, and attribution shapes.""" + with pytest.raises(ValueError): + SecurityConfig(admin_token="admin-only") + + assert SecurityConfig(auth_token="shared").readiness_profile()["auth_mode"] == "single_token" + assert SecurityConfig(admin_token="a", inference_token="i").readiness_profile()["auth_mode"] == "split_token" + assert SecurityConfig().readiness_profile()["auth_mode"] == "auth_not_configured" + + security = SecurityConfig(auth_token="shared", rate_limit_requests=2, rate_limit_window_seconds=1) + security._rate_buckets["client"] = (2, time.monotonic() - 1) + security.check_rate_limit("client") + assert security._rate_buckets["client"][0] == 1 + + assert _validate_mode("route") == "route" + for invalid in (None, 1, "invalid"): + with pytest.raises(RequestError): + _validate_mode(invalid) + + with pytest.raises(RequestError): + _validate_messages([]) + with pytest.raises(RequestError): + _validate_messages(["not-an-object"]) + with pytest.raises(RequestError): + _validate_messages([{"role": "owner", "content": "hello"}]) + assert _validate_messages([{"role": "user", "content": "hello", "ignored": True}]) == [ + {"role": "user", "content": "hello"} + ] + + assert _validate_attribution(None) is None + with pytest.raises(RequestError): + _validate_attribution("team-a") + with pytest.raises(RequestError): + _validate_attribution({"unknown_dimension": "x"}) + assert _validate_attribution({"team": 7, "provider": "mock"}) == {"team": "7", "provider": "mock"} + + assert _validate_routing(None) is None + with pytest.raises(RequestError): + _validate_routing("batch") + with pytest.raises(RequestError): + _validate_routing({"unexpected": True}) + with pytest.raises(RequestError): + _validate_routing({"channel": "async"}) + assert _validate_routing({"channel": "batch", "priority": "low"}) == {"channel": "batch", "priority": "low"} + + with pytest.raises(RequestError): + _coerce_json(b"[]") + assert _coerce_json(b'{"ok": true}') == {"ok": True} + + with pytest.raises(RequestError): + _validate_embeddings_inputs({}) + with pytest.raises(RequestError): + _validate_embeddings_inputs({"input": ["ok", 3]}) + assert _validate_embeddings_inputs({"input": "one"}) == ["one"] + + with pytest.raises(RequestError): + _embeddings_attribution({"metadata": "bad"}) + assert _embeddings_attribution( + { + "metadata": {"team": "metadata-team", "source": "ignored", "company": "acme", "group": ""}, + "attribution": {"team": "explicit-team"}, + } + ) == {"team": "explicit-team", "company": "acme"} + + with pytest.raises(RequestError): + _validate_batch_requests({}, False) + with pytest.raises(RequestError): + _validate_batch_requests({"requests": ["bad"]}, False) + batch = _validate_batch_requests( + { + "model": "default-model", + "attribution": {"company": "acme"}, + "requests": [ + { + "messages": [{"role": "user", "content": "hello"}], + "model": "child-model", + "attribution": {"team": "alpha"}, + "mode": "conduct", + } + ], + }, + False, + ) + assert batch[0].model == "child-model" + assert batch[0].attribution == {"company": "acme", "team": "alpha"} + assert batch[0].mode == "conduct" + + safe = _response_payload({"trace": [{"secret": "Bearer abcdefghijklmnopqrstuvwxyz"}], "nested": [{"trace": [1]}]}, False) + assert "trace" not in safe + + +def test_admin_get_matrix_covers_commercial_and_resource_routes() -> None: + """Authenticated admin GETs expose every buyer packet and resource outcome.""" + orchestrator = _build() + workflow = orchestrator.run([{"role": "user", "content": "build and verify the buyer packet"}], mode="route") + workflow_run_id = workflow["workflow_run_id"] + evaluation = orchestrator.run_evaluation(["replay buyer readiness"], mode="route") + evaluation_run_id = evaluation["evaluation_run_id"] + + commercial_paths = [ + "/api/v1/sales_readiness/latest", + "/api/v1/commercial_readiness/latest", + "/api/v1/buyer_evidence_manifests/latest", + "/api/v1/buyer_handoff_bundles/latest", + "/api/v1/saleability_decisions/latest", + "/api/v1/commercial_evidence_exports/latest", + "/api/v1/commercial_acceptance_checks/latest", + "/api/v1/commercial_release_candidates/latest", + "/api/v1/commercial_gap_registers/latest", + "/api/v1/commercial_procurement_readiness/latest", + "/api/v1/commercial_contract_readiness/latest", + "/api/v1/commercial_onboarding_readiness/latest", + "/api/v1/commercial_operations_readiness/latest", + "/api/v1/commercial_security_attestations/latest", + "/api/v1/commercial_value_readiness/latest", + "/api/v1/commercial_close_readiness/latest", + "/api/v1/commercial_go_to_market_readiness/latest", + "/api/v1/commercial_launch_readiness/latest", + "/api/v1/commercial_completion_scorecards/latest", + "/api/v1/commercial_buyer_acceptance_workflows/latest", + "/api/v1/commercial_demo_scenarios/latest", + "/api/v1/commercial_proposal_packets/latest", + "/api/v1/commercial_purchase_approval_packets/latest", + "/api/v1/commercial_due_diligence_rooms/latest", + "/api/v1/commercial_investment_committee_memos/latest", + ] + + with _running_server(orchestrator) as base_url: + for path in commercial_paths: + status, body, _headers = _request(base_url, "GET", path) + assert status == 200, (path, body) + assert isinstance(body, dict) + + success_paths = [ + "/api/v1/cost_attribution_dimensions", + "/api/v1/cost_reports/rollup?dimension=model_name", + "/api/v1/llm_usage_records?page_number=1&page_size=10", + "/api/v1/agent_pools?page_number=1&page_size=10", + "/api/v1/orchestration_policies/default_policy", + "/api/v1/analytics_snapshots/latest", + "/api/v1/spend_analytics/latest", + "/admin/state", + "/api/v1/workflow_runs?page_number=1&page_size=10", + f"/api/v1/workflow_runs/{workflow_run_id}", + f"/api/v1/access_reports/{workflow_run_id}", + f"/api/v1/evaluation_runs/{evaluation_run_id}", + "/api/v1/agent_pools/default_pool/worker_agents/general_agent", + "/api/v1/locale_bundles/en", + ] + for path in success_paths: + status, body, _headers = _request(base_url, "GET", path) + assert status == 200, (path, body) + + error_cases = [ + ("/api/v1/cost_reports/rollup?dimension=not-a-dimension", 400, "invalid_dimension"), + ("/api/v1/llm_usage_records?start=not-an-int", 400, "invalid_request"), + ("/api/v1/agent_pools?page_number=0", 400, "invalid_request"), + ("/api/v1/agent_pools?page_size=101", 400, "invalid_request"), + ("/api/v1/workflow_runs/missing", 404, "workflow_run_not_found"), + ("/api/v1/access_reports/missing", 404, "workflow_run_not_found"), + ("/api/v1/evaluation_runs/missing", 404, "evaluation_run_not_found"), + ("/api/v1/agent_pools/default_pool/worker_agents/missing", 404, "agent_not_found"), + ("/api/v1/agent_pools/default_pool/not_worker/missing", 400, "bad_path"), + ("/api/v1/locale_bundles/not-a-locale", 404, "locale_not_found"), + ("/api/v1/not-a-route", 404, "route_not_found"), + ("/api/v1/batch_routing_jobs/missing", 404, "batch_job_not_found"), + ] + for path, expected_status, expected_code in error_cases: + status, body, _headers = _request(base_url, "GET", path) + assert status == expected_status, (path, body) + assert isinstance(body, dict) + assert body["error"]["code"] == expected_code + + +def test_agent_crud_batch_and_post_error_matrix() -> None: + """POST/PATCH/DELETE cover operator CRUD, batch surfaces, and HTTP-safe failures.""" + orchestrator = _build() + with _running_server(orchestrator, max_body_bytes=1024) as base_url: + create_payload = { + "id": "coverage_agent", + "model": "mock-coverage", + "base_url": "mock://coverage", + "tags": ["reasoning"], + "priority": 5, + } + status, body, _ = _request(base_url, "POST", "/api/v1/agent_pools/default_pool/worker_agents", create_payload) + assert status == 201 + assert body["id"] == "coverage_agent" + + status, body, _ = _request( + base_url, + "PATCH", + "/api/v1/agent_pools/default_pool/worker_agents/coverage_agent", + {"priority": 9, "tags": ["reasoning", "review"]}, + ) + assert status == 200 + assert body["priority"] == 9 + + status, body, _ = _request(base_url, "DELETE", "/api/v1/agent_pools/default_pool/worker_agents/coverage_agent") + assert status == 200 + + status, body, _ = _request( + base_url, + "PATCH", + "/api/v1/agent_pools/default_pool/worker_agents/missing", + {"priority": 2}, + ) + assert status == 404 + assert body["error"]["code"] == "resource_not_found" + + status, body, _ = _request(base_url, "DELETE", "/api/v1/agent_pools/default_pool/worker_agents/missing") + assert status == 404 + assert body["error"]["code"] == "resource_not_found" + + for method, path in [ + ("PATCH", "/api/v1/agent_pools/default_pool/extra/worker_agents/missing"), + ("DELETE", "/api/v1/agent_pools/default_pool/extra/worker_agents/missing"), + ]: + status, body, _ = _request(base_url, method, path, {} if method == "PATCH" else None) + assert status == 400 + assert body["error"]["code"] == "bad_path" + + status, body, _ = _request( + base_url, + "PATCH", + "/api/v1/agent_pools/default_pool/worker_agents/general_agent", + {"unknown": True}, + ) + assert status == 400 + assert body["error"]["code"] == "unknown_fields" + + status, body, _ = _request( + base_url, + "POST", + "/api/v1/agent_pools/default_pool/extra/worker_agents", + create_payload, + ) + assert status == 400 + assert body["error"]["code"] == "bad_path" + + status, body, _ = _request(base_url, "POST", "/admin/simulate", {"prompt": 3}) + assert status == 400 + assert body["error"]["code"] == "invalid_request" + + status, body, _ = _request(base_url, "POST", "/admin/simulate", {"prompt": "simulate", "mode": "route"}) + assert status == 200 + assert isinstance(body, dict) + + status, body, _ = _request(base_url, "POST", "/api/v1/workflow_runs", {"prompt_text": "run this", "run_mode": "route"}) + assert status == 201 + assert body["mode"] == "route" + + status, body, _ = _request(base_url, "POST", "/api/v1/workflow_runs", {"prompt_text": ""}) + assert status == 400 + assert body["error"]["code"] == "invalid_request" + + status, body, _ = _request(base_url, "POST", "/api/v1/evaluation_runs", {"prompt_text": "evaluate this", "run_mode": "route"}) + assert status == 201 + assert body["prompt_count"] == 1 + + status, body, _ = _request(base_url, "POST", "/api/v1/evaluation_runs", {"prompts": []}) + assert status == 400 + assert body["error"]["code"] == "invalid_request" + + status, body, _ = _request( + base_url, + "POST", + "/v1/batch/embeddings", + {"model": "mock-embedding", "input": ["alpha", "beta"], "metadata": {"team": "coverage"}}, + ) + assert status in {200, 202} + assert isinstance(body, dict) + batch_id = body["batch_id"] + poll_status, poll_body, _ = _request(base_url, "GET", f"/v1/batch/embeddings/{batch_id}") + assert poll_status == 200 + assert poll_body["batch_id"] == batch_id + + status, body, _ = _request(base_url, "GET", "/v1/batch/embeddings/missing") + assert status == 404 + assert body["error"]["code"] == "embeddings_batch_not_found" + + status, body, _ = _request( + base_url, + "POST", + "/api/v1/batch_routing_jobs", + { + "model": "mock-generalist", + "requests": [ + {"messages": [{"role": "user", "content": "batch hello"}], "mode": "route"} + ], + }, + ) + assert status == 201 + job_id = body["job_id"] + + status, body, _ = _request(base_url, "GET", f"/api/v1/batch_routing_jobs/{job_id}") + assert status == 200 + assert body["job_id"] == job_id + + status, body, _ = _request(base_url, "POST", f"/api/v1/batch_routing_jobs/{job_id}/results", {}) + assert status == 200 + assert isinstance(body, dict) + + status, body, _ = _request(base_url, "POST", "/api/v1/batch_routing_jobs/missing/results", {}) + assert status == 404 + assert body["error"]["code"] == "batch_job_not_found" + + status, body, _ = _request(base_url, "POST", "/v1/responses", {"model": "mock-generalist", "input": "respond"}) + assert status == 200 + assert isinstance(body, dict) + + status, body, _ = _request(base_url, "POST", "/not-a-route", {}) + assert status == 404 + assert body["error"]["code"] == "route_not_found" + + status, body, _ = _request(base_url, "POST", "/admin/simulate", raw_body=b"{not-json") + assert status == 400 + assert body["error"]["code"] == "invalid_json" + + status, body, _ = _request( + base_url, + "POST", + "/admin/simulate", + {"prompt": "text"}, + content_type="text/plain", + ) + assert status == 415 + assert body["error"]["code"] == "unsupported_media_type" + + status, body, _ = _request( + base_url, + "POST", + "/admin/simulate", + raw_body=b"x" * 1025, + extra_headers={"content-type": "application/json"}, + ) + assert status == 413 + assert body["error"]["code"] == "request_too_large" + + +def test_document_surfaces_and_buffered_stream_preserve_content_types() -> None: + """OpenAPI, admin HTML, and buffered SSE return their public wire formats.""" + orchestrator = _build() + with _running_server(orchestrator) as base_url: + status, body, headers = _request(base_url, "GET", "/openapi.json", token=None) + assert status == 200 + assert isinstance(body, dict) + assert body["info"]["title"] == "Contextual Orchestrator API" + assert headers["content-type"].startswith("application/json") + + status, body, headers = _request(base_url, "GET", "/admin") + assert status == 200 + assert "Contextual Orchestrator" in body + assert headers["content-type"] == "text/html; charset=utf-8" + + status, body, headers = _request( + base_url, + "POST", + "/v1/chat/completions", + { + "model": "contextual-orchestrator", + "messages": [{"role": "user", "content": "plan, implement, and verify this change"}], + "mode": "conduct", + "stream": True, + }, + ) + assert status == 200 + assert headers["content-type"] == "text/event-stream; charset=utf-8" + assert '"object": "chat.completion.chunk"' in body + assert body.endswith("data: [DONE]\n\n") + + +def test_live_stream_reports_a_terminal_error_after_headers_are_sent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A route stream failure terminates with an error frame and releases the slot.""" + orchestrator = _build() + + def failing_stream(_messages, workflow_run_id=None): + yield "partial" + raise RuntimeError("provider disconnected") + + monkeypatch.setattr(orchestrator, "stream_route", failing_stream) + with _running_server(orchestrator) as base_url: + status, body, headers = _request( + base_url, + "POST", + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "stream this answer"}], + "mode": "route", + "stream": True, + }, + ) + + assert status == 200 + assert headers["content-type"] == "text/event-stream; charset=utf-8" + assert '"content": "partial"' in body + assert '"finish_reason": "error"' in body + assert body.endswith("data: [DONE]\n\n") + + +def test_http_method_error_boundaries_return_stable_statuses( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """PATCH, DELETE, GET, and POST convert domain and unexpected failures safely.""" + orchestrator = _build() + with _running_server(orchestrator) as base_url: + status, body, _ = _request(base_url, "PATCH", "/api/v1/not-a-route", {}) + assert status == 404 + assert body["error"]["code"] == "route_not_found" + + status, body, _ = _request(base_url, "DELETE", "/api/v1/not-a-route") + assert status == 404 + assert body["error"]["code"] == "route_not_found" + + status, body, _ = _request( + base_url, + "PATCH", + "/api/v1/agent_pools/default_pool/worker_agents/general_agent", + {"status": "not-a-status"}, + ) + assert status == 400 + assert body["error"]["code"] == "invalid_request" + + monkeypatch.setattr(orchestrator, "remove_agent", lambda *_args: (_ for _ in ()).throw(ValueError("last agent"))) + status, body, _ = _request( + base_url, + "DELETE", + "/api/v1/agent_pools/default_pool/worker_agents/general_agent", + ) + assert status == 400 + assert body["error"]["code"] == "invalid_request" + + failure_cases = [ + ("admin_state", "GET", "/admin/state", None), + ( + "patch_agent", + "PATCH", + "/api/v1/agent_pools/default_pool/worker_agents/general_agent", + {"priority": 3}, + ), + ( + "remove_agent", + "DELETE", + "/api/v1/agent_pools/default_pool/worker_agents/general_agent", + None, + ), + ( + "add_agent", + "POST", + "/api/v1/agent_pools/default_pool/worker_agents", + {"id": "new_agent", "model": "mock-new", "base_url": "mock://new"}, + ), + ] + for method_name, http_method, path, payload in failure_cases: + monkeypatch.setattr( + orchestrator, + method_name, + lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unexpected failure")), + ) + status, body, _ = _request(base_url, http_method, path, payload) + assert status == 500, (method_name, body) + assert body["error"]["code"] == "internal_error" + + +def test_serve_starts_the_built_server(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """The blocking serve entrypoint announces its address and starts serving.""" + started: list[bool] = [] + + class FakeServer: + def serve_forever(self) -> None: + started.append(True) + + monkeypatch.setattr(server_module, "build_server", lambda *_args, **_kwargs: FakeServer()) + server_module.serve(_build(), host="127.0.0.1", port=8765) + + assert started == [True] + assert capsys.readouterr().out == "listening on http://127.0.0.1:8765\n" diff --git a/tests/test_token_counting.py b/tests/test_token_counting.py new file mode 100644 index 000000000..908fd6094 --- /dev/null +++ b/tests/test_token_counting.py @@ -0,0 +1,79 @@ +"""Token-counting seam: heuristic estimator, pg_tiktoken adapter, and factory. + +Runs entirely on the dependency-free heuristic path plus a fake pg_tiktoken +counter that records its calls — no Postgres or ``pg_llm_batch`` install. +""" + +from __future__ import annotations + +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.token_counting import ( # noqa: E402 + HeuristicTokenCounter, + PgTiktokenAdapter, + build_token_counter, +) + + +def test_heuristic_counts_words_with_bpe_expansion() -> None: + """Two word units expand to ceil(2 * 1.3) == 3 tokens.""" + assert HeuristicTokenCounter().count_text("hello world") == 3 + + +def test_heuristic_empty_text_is_zero() -> None: + """Empty text counts as zero tokens.""" + assert HeuristicTokenCounter().count_text("") == 0 + + +def test_heuristic_whitespace_only_text_is_zero() -> None: + """Non-empty text with no word/punctuation units counts as zero tokens.""" + assert HeuristicTokenCounter().count_text(" \t\n ") == 0 + + +def test_heuristic_count_messages_adds_per_message_framing() -> None: + """Each message contributes its content tokens plus fixed framing overhead.""" + counter = HeuristicTokenCounter() + messages = [{"role": "user", "content": "hi there"}, "not_a_dict"] + assert counter.count_messages(messages) == 3 + 3 + 3 + + +class _FakePgTokenCounter: + """Stand-in for pg_llm_batch.TokenCounter; records each call for assertions.""" + + def __init__(self) -> None: + """Start with an empty call log.""" + self.calls: list[tuple[str, str]] = [] + + def count_tokens(self, text: str, model: str) -> int: + """Record (text, model) and return a deterministic whitespace-split count.""" + self.calls.append((text, model)) + return len(text.split()) + + +def test_pg_adapter_count_text_delegates_text_and_model() -> None: + """count_text forwards both the text and the model to the backend.""" + backend = _FakePgTokenCounter() + adapter = PgTiktokenAdapter(backend) + assert adapter.count_text("one two three", "gpt_example") == 3 + assert backend.calls == [("one two three", "gpt_example")] + + +def test_pg_adapter_count_messages_forwards_content_and_model() -> None: + """count_messages forwards each message's content and the model per call.""" + backend = _FakePgTokenCounter() + adapter = PgTiktokenAdapter(backend) + messages = [{"content": "one two"}, {"content": "three"}, 42] + assert adapter.count_messages(messages, "gpt_example") == 3 + assert backend.calls == [ + ("one two", "gpt_example"), + ("three", "gpt_example"), + ("", "gpt_example"), + ] + + +def test_build_token_counter_without_dsn_is_heuristic() -> None: + """With no DSN the factory returns the dependency-free heuristic counter.""" + assert isinstance(build_token_counter(), HeuristicTokenCounter) diff --git a/tests/test_true_streaming.py b/tests/test_true_streaming.py index ef451955b..8a7db6ffd 100644 --- a/tests/test_true_streaming.py +++ b/tests/test_true_streaming.py @@ -14,6 +14,8 @@ import threading import urllib.request +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 @@ -24,13 +26,17 @@ class _FakeSSEProvider: """Emits a fixed list of raw SSE frame strings at POST /chat/completions.""" - def __init__(self, frames: list[str]) -> None: + def __init__( + self, + frames: list[str], + content_type: str = "text/event-stream", + ) -> None: class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 length = int(self.headers.get("content-length", 0)) self.rfile.read(length) self.send_response(200) - self.send_header("content-type", "text/event-stream") + self.send_header("content-type", content_type) self.end_headers() for frame in frames: self.wfile.write(frame.encode("utf-8")) @@ -74,6 +80,93 @@ def test_stream_send_parses_real_provider_sse() -> None: assert "".join(deltas) == "Hello streamed world" +def test_stream_send_rejects_eof_before_done_marker() -> None: + """An interrupted OpenAI-compatible stream cannot be reported as complete.""" + with _FakeSSEProvider([_delta("partial")]) as provider: + client = ModelClient() + agent = ModelAgent( + "worker_agent", + "gpt-x", + base_url=provider.base_url, + api_key_env="UNSET_KEY_ENV", + ) + with pytest.raises(RuntimeError, match="terminated before.*DONE"): + list(client._stream_send(agent, {"model": "gpt-x", "stream": True})) + + +def test_stream_send_rejects_malformed_data_event() -> None: + """Malformed provider data frames fail closed instead of disappearing silently.""" + frames = ["data: {not-json}\n\n", "data: [DONE]\n\n"] + with _FakeSSEProvider(frames) as provider: + client = ModelClient() + agent = ModelAgent( + "worker_agent", + "gpt-x", + base_url=provider.base_url, + api_key_env="UNSET_KEY_ENV", + ) + with pytest.raises(RuntimeError, match="malformed provider stream event"): + list(client._stream_send(agent, {"model": "gpt-x", "stream": True})) + + +def test_stream_send_rejects_non_event_stream_response() -> None: + """A 200 response with the wrong media type cannot masquerade as SSE success.""" + frames = [_delta("should-not-publish"), "data: [DONE]\n\n"] + with _FakeSSEProvider(frames, content_type="application/json") as provider: + client = ModelClient() + agent = ModelAgent( + "worker_agent", + "gpt-x", + base_url=provider.base_url, + api_key_env="UNSET_KEY_ENV", + ) + with pytest.raises(RuntimeError, match="event-stream content type"): + list(client._stream_send(agent, {"model": "gpt-x", "stream": True})) + + +class _IterableStreamResponse: + """Lightweight context response for exercising the documented test-double seam.""" + + def __init__(self, frames: list[bytes]) -> None: + self.frames = frames + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + def __iter__(self): + return iter(self.frames) + + +def test_stream_send_stops_at_done_for_lightweight_response_double() -> None: + """The non-HTTP seam emits no provider data after the terminal marker.""" + client = ModelClient() + client._open_provider = lambda _request: _IterableStreamResponse( # type: ignore[method-assign] + [ + _delta("accepted").encode(), + b"data: [DONE]\n\n", + _delta("must-not-escape").encode(), + ] + ) + agent = ModelAgent("worker_agent", "gpt-x", base_url="https://provider.example") + + assert list(client._stream_send(agent, {"model": "gpt-x", "stream": True})) == ["accepted"] + + +def test_stream_send_rejects_malformed_event_from_lightweight_response_double() -> None: + """The non-HTTP seam enforces the same fail-closed JSON contract as live HTTP.""" + client = ModelClient() + client._open_provider = lambda _request: _IterableStreamResponse( # type: ignore[method-assign] + [b"data: {not-json}\n\n", b"data: [DONE]\n\n"] + ) + agent = ModelAgent("worker_agent", "gpt-x", base_url="https://provider.example") + + with pytest.raises(RuntimeError, match="malformed provider stream event"): + list(client._stream_send(agent, {"model": "gpt-x", "stream": True})) + + def test_stream_chat_mock_yields_chunks() -> None: client = ModelClient() agent = ModelAgent("general_agent", "mock-model") # base_url defaults to mock://local