From 2f1ecd19662c03da1877b6edd29a23d423afa988 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 16:23:00 +0000 Subject: [PATCH 1/5] feat: seed org provider catalog and OpenCode sidecar Register NVIDIA NIM, OpenAI, OpenRouter, and Bytez credentials into the KV, compose a production agent pool (no GitHub Models), and fail closed on missing keys, 429 failover, and malformed upstream responses. Co-authored-by: Seongho Bae --- .github/workflows/opencode-sidecar.yml | 97 +++++++ AGENTS.md | 29 +- CHANGELOG.md | 28 ++ CLAUDE.md | 6 +- README.md | 11 + conductor/product.md | 2 + conductor/tracks.md | 1 + contextual_orchestrator/__main__.py | 62 +++++ contextual_orchestrator/orchestrator.py | 96 ++++++- contextual_orchestrator/provider_catalog.py | 271 ++++++++++++++++++ docs/architecture.md | 3 + docs/doctoring/provider-catalog.md | 77 ++++++ docs/fuzzing.md | 3 + docs/kv-credentials.md | 23 +- docs/library_research.md | 10 + docs/opencode-sidecar.md | 118 ++++++++ docs/papers/README.md | 6 + examples/agents.production.json | 85 ++++++ fuzz/targets.py | 13 +- tests/fuzz/test_fuzz_properties.py | 7 + tests/test_catalog_bootstrap.py | 290 ++++++++++++++++++++ tests/test_conventions.py | 10 +- tests/test_opencode_sidecar_contract.py | 88 ++++++ tests/test_provider_catalog.py | 156 +++++++++++ tests/test_provider_catalog_robustness.py | 284 +++++++++++++++++++ 25 files changed, 1743 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/opencode-sidecar.yml create mode 100644 CHANGELOG.md create mode 100644 contextual_orchestrator/provider_catalog.py create mode 100644 docs/doctoring/provider-catalog.md create mode 100644 docs/opencode-sidecar.md create mode 100644 examples/agents.production.json create mode 100644 tests/test_catalog_bootstrap.py create mode 100644 tests/test_opencode_sidecar_contract.py create mode 100644 tests/test_provider_catalog.py create mode 100644 tests/test_provider_catalog_robustness.py diff --git a/.github/workflows/opencode-sidecar.yml b/.github/workflows/opencode-sidecar.yml new file mode 100644 index 000000000..ebfc9a8ba --- /dev/null +++ b/.github/workflows/opencode-sidecar.yml @@ -0,0 +1,97 @@ +name: OpenCode sidecar + +# Deploy/CI sidecar only. Does not run on pull_request so app tests and the +# required Security workflow stay secret-free (see tests.yml / security.yml). +on: + workflow_dispatch: + workflow_call: + secrets: + NVIDIA_NIM_API_KEY: + required: false + NVIDIA_NIM_API_KEY_SUB: + required: false + OPENAI_API_KEY: + required: false + OPENROUTER_API_KEY: + required: false + BYTEZ_API_KEY: + required: false + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: opencode-sidecar-${{ github.ref }} + cancel-in-progress: true + +jobs: + seed_and_serve: + name: Seed KV and serve loopback sidecar + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install package + run: | + python -m pip install --require-hashes -r requirements.lock + python -m pip install --no-deps -e . + + - name: Seed org credentials into this-job report + env: + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} + run: | + python -m contextual_orchestrator seed-provider-catalog \ + --from-env --skip-missing \ + --agents examples/agents.production.json \ + --agents-db "$RUNNER_TEMP/agents.db" \ + --discover-models | tee "$RUNNER_TEMP/seed-report.json" + + - name: Serve loopback OpenAI-compatible API and smoke curl + env: + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} + run: | + set -euo pipefail + CONTEXTUAL_ORCHESTRATOR_TOKEN="$(python -c 'import secrets; print(secrets.token_urlsafe(32))')" + export CONTEXTUAL_ORCHESTRATOR_TOKEN + python -m contextual_orchestrator --serve \ + --seed-from-env \ + --agents examples/agents.production.json \ + --agents-db "$RUNNER_TEMP/agents.db" \ + --host 127.0.0.1 \ + --port 8000 \ + --auth-token "$CONTEXTUAL_ORCHESTRATOR_TOKEN" & + server_pid=$! + trap 'kill "$server_pid" 2>/dev/null || true' EXIT + for _ in 1 2 3 4 5 6 7 8 9 10; do + if curl -sf http://127.0.0.1:8000/healthz >/dev/null; then + break + fi + sleep 1 + done + registered="$(python -c 'import json,sys; print(len(json.load(open(sys.argv[1]))["registered_credentials"]))' "$RUNNER_TEMP/seed-report.json")" + if [ "$registered" = "0" ]; then + echo "no provider secrets in this job; skip live chat smoke (fail-closed, no GitHub Models fallback)" + exit 0 + fi + curl -sS --fail http://127.0.0.1:8000/v1/chat/completions \ + -H "authorization: Bearer $CONTEXTUAL_ORCHESTRATOR_TOKEN" \ + -H "content-type: application/json" \ + -d '{"model":"contextual-orchestrator","messages":[{"role":"user","content":"Write one sentence."}]}' diff --git a/AGENTS.md b/AGENTS.md index 7ced99577..c432f7570 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,23 +56,30 @@ push or open a PR. - The reference implementation is xtrmLLMBatchPython's pgcrypto-encrypted Postgres credential registry (`get_credential(name)`); reuse that pattern (a DB-backed KV is fine) unless a dedicated KV is adopted. -- **Known deviation to migrate:** this repo currently resolves provider API - keys from env — `ModelClient` reads `os.environ.get(agent.api_key_env)` in - `contextual_orchestrator/orchestrator.py` (and `CONTEXTUAL_ORCHESTRATOR_*` - tokens in `__main__.py`). Move these to KV-backed reads; keep env only as the - bootstrap path that seeds the KV. +- Runtime provider keys resolve through `get_credential` / the KV registry. + Env is bootstrap transport only (`register-credential --from-env` / + `seed-provider-catalog --from-env` / `--seed-from-env` on serve). The five + org Actions secrets are `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, + `OPENAI_API_KEY`, `OPENROUTER_API_KEY`, and `BYTEZ_API_KEY`. A missing secret + skips that upstream; it must not crash the pool. +- ContextualWisdomLab **no longer uses GitHub Models**. Do not add + `COPILOT_GITHUB_TOKEN`, `models.github.ai`, `gpt-5.6-luna`, or `gpt-5.6-terra`. ### This repo: the org LLM gateway - `contextual-orchestrator` is the org's **LLM-communication hub** — the - OpenAI-compatible front door consumed by **gyeot** and **scopeweave**. + OpenAI-compatible front door consumed by **gyeot**, **scopeweave**, + **OpenCode**, and **Strix**. - **Direction:** grow it toward a **LiteLLM-class multi-provider gateway**. The org is open to a **Rust/Python hybrid** to cut overhead. -- Its `ModelClient` currently reads `os.environ.get(agent.api_key_env)` — this - is the KV-principle deviation above. Resolve the API key (including the org - `OPENAI_API_KEY`) from the **KV / credential registry**, not env. -- The **OpenCode review pipeline is separate** and stays on **GitHub Models** — - do not change it. +- `ModelClient` resolves the API key from the **KV / credential registry** via + `get_credential` (including the org `OPENAI_API_KEY` and NIM / OpenRouter / + Bytez keys). Env is never the request-time source. +- OpenCode/Strix should call this process as **one** OpenAI-compatible + provider: `baseURL http://127.0.0.1:8000/v1`, model `contextual-orchestrator` + (see `docs/opencode-sidecar.md`). The org-central review workflow lives in + `ContextualWisdomLab/.github` and should consume this sidecar — do not + reintroduce GitHub Models there from this repo. ### This repo's role in the ecosystem diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..e432ef354 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,28 @@ +# Changelog + +## Unreleased + +### Added + +- Production agent catalog (`examples/agents.production.json`) for NVIDIA NIM + (primary + secondary Nemotron Super 49B / 120B), OpenAI, OpenRouter, and + Bytez. Capability tags cover coding, review, and reasoning so Fugu route vs + Conductor/TRINITY conduct can pick workers. GitHub Models, Copilot tokens, + `gpt-5.6-luna`, and `gpt-5.6-terra` are rejected. +- `seed-provider-catalog` CLI and `--seed-from-env` serve flag register the five + org Actions secrets (`NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, + `OPENAI_API_KEY`, `OPENROUTER_API_KEY`, `BYTEZ_API_KEY`) into the KV. A + missing secret skips that upstream. Optional `GET /v1/models` discovery + appends chat models; providers without a list API keep the static seed + (`docs/doctoring/provider-catalog.md`). +- OpenCode/Strix sidecar contract: loopback `http://127.0.0.1:8000/v1`, model + `contextual-orchestrator` (`.github/workflows/opencode-sidecar.yml`, + `docs/opencode-sidecar.md`). App tests and Security stay secret-free. + +### Changed + +- Unconfigured remote workers are skipped at select/failover time. When every + provider credential is missing, routing raises `NotConfigured` and does not + fall back to GitHub Models. +- Malformed upstream chat.completion bodies raise `ProviderResponseError` so + the gateway failovers or returns a JSON error instead of crashing. diff --git a/CLAUDE.md b/CLAUDE.md index f893b5f7b..a965d356c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - **Security gate**: every PR to `main` runs the required Security workflow. A failing Trivy or pip-audit job is a real finding — remediate by bumping the dependency and regenerating `requirements.lock`; never weaken, `continue-on-error`, or disable the gate. - **KV, not env**: runtime config and provider secrets are resolved from the KV credential registry (`get_credential`), never `os.getenv` at request time. Env is only bootstrap transport into the KV (see `docs/kv-credentials.md`). -- **Org role**: this repo is the org's LLM gateway (cost optimizer + sync/batch routing + upstream load balancing, LiteLLM-plus scope), consumed by `gyeot` and `scopeweave`. The OpenCode review pipeline is separate, stays on GitHub Models, and must not be changed. +- **Org role**: this repo is the org's LLM gateway (cost optimizer + sync/batch routing + upstream load balancing, LiteLLM-plus scope), consumed by `gyeot`, `scopeweave`, OpenCode, and Strix. ContextualWisdomLab no longer uses GitHub Models; OpenCode should call `http://127.0.0.1:8000/v1` model `contextual-orchestrator` (see `docs/opencode-sidecar.md`). - **Research grounding**: substantive feature/process PRs should attach the relevant papers (PDF when redistribution is permissible, otherwise cite + link + summary) under `docs/papers/` with full citations. This file complements AGENTS.md with commands and architecture; where they differ, AGENTS.md wins. @@ -55,6 +55,8 @@ python -m contextual_orchestrator --eval "prompt one" "prompt two" # Seed a provider credential into the KV at bootstrap echo "$OPENAI_API_KEY" | python -m contextual_orchestrator register-credential --name OPENAI_API_KEY --value-stdin +python -m contextual_orchestrator seed-provider-catalog --from-env --skip-missing \ + --agents examples/agents.production.json --agents-db /tmp/agents.db # Reproduce the Trivy security gate locally (against the merge result) trivy --download-db-only @@ -91,7 +93,7 @@ A stdlib-Python lab implementing a single OpenAI-compatible API that routes, del - `credentials.py` / `kv_config.py` — the KV seam: `get_credential`/`register_credential` over pluggable backends (`InMemoryCredentialBackend` default; pgcrypto-encrypted `PostgresCredentialBackend`, selected via `CONTEXTUAL_ORCHESTRATOR_KV_BACKEND`). - `cost_ledger.py` / `cost_router.py` / `batch_routing.py` / `token_counting.py` — the cost-review + routing hub: prompt-safe usage ledger with seven attribution dimensions, `RoutingPolicy` (sync vs batch from request hints + KV thresholds), and the [pg-llm-batch](https://github.com/ContextualWisdomLab/pg-llm-batch) batch/embeddings backends (a local in-process backend keeps the standalone path working with no external service). - `api_contract.py` / `conventions.py` — API-shape and naming-rule enforcement helpers. -- `__main__.py` — the single entry point: CLI completion, `--serve`, `--eval`, and the `register-credential` bootstrap subcommand. +- `__main__.py` — the single entry point: CLI completion, `--serve`, `--eval`, `register-credential`, and `seed-provider-catalog` / `--seed-from-env`. Agent pools are **data, not code**: `examples/agents.mock.json` and `examples/agents.openai.json`. State is in-memory by default; `--state-db PATH` persists runs/audit/analytics to sqlite. diff --git a/README.md b/README.md index 65f57dd4c..70a020532 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,13 @@ HTTP serving is hardened for local lab use: - Response caching is off by default. Pass `--cache-ttl SECONDS` to serve identical requests (same messages + mode) from an in-memory TTL+LRU cache and skip the provider calls; `0` disables it. - `ModelClient.batch_chat(agent, {custom_id: messages})` runs many requests through the provider's Batch API (async, 24h completion window, typically ~50% cheaper) — suited to evaluation/benchmark workloads, not latency-sensitive chat. The mock path answers synchronously. +Production (not mock-only) seed: [examples/agents.production.json](examples/agents.production.json) — NVIDIA NIM primary/secondary (Nemotron Super 49B / 120B), OpenAI, OpenRouter, and Bytez. OpenCode/Strix should call `http://127.0.0.1:8000/v1` with model `contextual-orchestrator` ([docs/opencode-sidecar.md](docs/opencode-sidecar.md)). GitHub Models are not in the catalog. + +```bash +python -m contextual_orchestrator seed-provider-catalog --from-env --skip-missing \ + --agents examples/agents.production.json --agents-db /tmp/agents.db +``` + Use real workers by replacing `mock://` agents with OpenAI-compatible endpoints. Provider secrets are resolved from a KV credential registry via `get_credential`, never from `os.getenv` at request time (see [docs/kv-credentials.md](docs/kv-credentials.md)): ```json @@ -252,6 +259,10 @@ python -m pip install --require-hashes -r requirements.lock python -m pip install --no-deps -e . python tests/test_self_check.py python tests/test_paper_contracts.py +python tests/test_provider_catalog.py +python tests/test_catalog_bootstrap.py +python tests/test_provider_catalog_robustness.py +python tests/test_opencode_sidecar_contract.py python tests/test_admin_contract.py python tests/test_conventions.py python tests/test_api_contract.py diff --git a/conductor/product.md b/conductor/product.md index 3092bda2a..cfd55f4c7 100644 --- a/conductor/product.md +++ b/conductor/product.md @@ -39,5 +39,7 @@ Provide one API and one domain model: - Training a learned coordinator. - Claiming compatibility with or ownership of any vendor model. - Adding provider SDKs before stdlib HTTP proves insufficient. +- Reintroducing GitHub Models (`COPILOT_GITHUB_TOKEN`, `gpt-5.6-luna` / `terra`). + OpenCode and Strix call this process as one OpenAI-compatible provider. See `docs/product_planning.md` for the paper-grounded product plan. diff --git a/conductor/tracks.md b/conductor/tracks.md index 968c08ef8..af49e5170 100644 --- a/conductor/tracks.md +++ b/conductor/tracks.md @@ -4,3 +4,4 @@ |---|---|---| | 001-paper-grounded-orchestrator | active | Implement the source-backed orchestration contract with TDD, DDD, and CDD | | 002-enterprise-design-foundation | active | Add paper-grounded screen design, user stories, REST API, code/DB conventions, and i18n | +| 003-org-provider-catalog | active | Seed NIM/OpenAI/OpenRouter/Bytez into the KV, OpenCode sidecar, exception-robust failover | diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 5f68c3b74..4400bf44b 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -9,6 +9,7 @@ from .credentials import register_credential from .orchestrator import ModelClient, TaskOrchestrator, load_agents +from .provider_catalog import seed_provider_catalog from .server import SecurityConfig, serve @@ -55,11 +56,54 @@ def _register_credential_command(argv: list[str]) -> None: print(json.dumps({"registered": args.name, "backend": "kv"}, ensure_ascii=False)) +def _seed_provider_catalog_command(argv: list[str]) -> None: + """Register the five org secrets (skip missing) and compose the production catalog.""" + parser = argparse.ArgumentParser( + prog="python -m contextual_orchestrator seed-provider-catalog", + description="Bootstrap org provider credentials and the production agent catalog.", + ) + parser.add_argument( + "--agents", + default="examples/agents.production.json", + help="Production seed JSON (default: examples/agents.production.json).", + ) + parser.add_argument("--agents-db", default=None, help="Optional sqlite path to persist the ready pool.") + parser.add_argument( + "--from-env", + action="store_true", + help="Bootstrap transport: read NVIDIA_NIM_API_KEY, NVIDIA_NIM_API_KEY_SUB, " + "OPENAI_API_KEY, OPENROUTER_API_KEY, and BYTEZ_API_KEY from the process env.", + ) + parser.add_argument( + "--skip-missing", + action="store_true", + default=True, + help="Skip a secret that is unset and keep serving the remaining providers (default).", + ) + parser.add_argument( + "--discover-models", + action="store_true", + help="GET /v1/models for each registered credential and append discovered chat models.", + ) + args = parser.parse_args(argv) + if not args.from_env: + parser.error("--from-env is required so secrets enter the KV only as bootstrap transport") + report = seed_provider_catalog( + seed_path=args.agents, + agents_db=args.agents_db, + discover=args.discover_models, + ) + print(json.dumps(report, ensure_ascii=False, indent=2)) + + def main() -> None: """Parse CLI options and run bootstrap, prompt completion, or the HTTP server.""" if len(sys.argv) > 1 and sys.argv[1] == "register-credential": _register_credential_command(sys.argv[2:]) return + if len(sys.argv) > 1 and sys.argv[1] == "seed-provider-catalog": + _seed_provider_catalog_command(sys.argv[2:]) + return parser = argparse.ArgumentParser(description="Route or conduct chat requests across model agents.") parser.add_argument("prompt", nargs="?", help="User prompt for CLI mode.") @@ -80,6 +124,17 @@ def main() -> None: help="Base URL of a Clearfolio deployment to use as the admin document viewer (default: disabled).") parser.add_argument("--agents-db", default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_AGENTS_DB") or None, help="Optional sqlite path so runtime agent-pool changes (add/patch/remove) survive restarts.") + parser.add_argument( + "--seed-from-env", + action="store_true", + help="Before serving, register the five org secrets from env into this process KV " + "(CI/memory backend; postgres deploys should run seed-provider-catalog instead).", + ) + parser.add_argument( + "--discover-models", + action="store_true", + help="With --seed-from-env, also GET /v1/models for each registered credential.", + ) parser.add_argument("--provider-ca-bundle", default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_PROVIDER_CA_BUNDLE") or None, help="Path to a CA bundle used to verify provider TLS (e.g. a corporate gateway root).") parser.add_argument("--insecure-skip-tls-verify", action="store_true", @@ -94,6 +149,13 @@ def main() -> None: help="Measure orchestration vs a single-worker baseline on these prompts and print the report.") args = parser.parse_args() + if args.seed_from_env: + seed_provider_catalog( + seed_path=args.agents, + agents_db=args.agents_db, + discover=args.discover_models, + ) + client = ModelClient(ca_bundle=args.provider_ca_bundle, verify_tls=not args.insecure_skip_tls_verify) orchestrator = TaskOrchestrator( load_agents(args.agents), diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 0097b722e..9057350a3 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -82,6 +82,10 @@ class ModelAgent: def __post_init__(self) -> None: require_object_name(self.id, "agent.id") + if not catalog_allows_fields(self.base_url, self.model, self.api_key_env or self.credential_key): + raise ValueError( + "GitHub Models and Copilot tokens are not permitted in the agent catalog" + ) def to_config(self) -> dict[str, Any]: """Round-trippable agent configuration (from_dict(to_config(a)) == a).""" @@ -189,6 +193,38 @@ def as_dict(self) -> dict[str, Any]: # is a caller or configuration error and must not be retried. TRANSIENT_HTTP_STATUS = frozenset({408, 409, 425, 429, 500, 502, 503, 504}) +# GitHub Models is retired for this org. These markers fail-closed at agent +# construction so a catalog or runtime add cannot silently reintroduce them. +FORBIDDEN_HOST_MARKERS = frozenset( + { + "models.github.ai", + "models.inference.ai.azure.com", + "api.githubcopilot.com", + "models.github.com", + } +) +FORBIDDEN_MODEL_MARKERS = frozenset({"gpt-5.6-luna", "gpt-5.6-terra"}) +FORBIDDEN_CREDENTIAL_NAMES = frozenset({"COPILOT_GITHUB_TOKEN"}) + + +def catalog_allows_fields(base_url: str, model: str, credential_name: str) -> bool: + """Return True when an agent record is not a retired GitHub Models/Copilot target.""" + host = (urlparse(base_url).hostname or "").lower() + if any(marker in host for marker in FORBIDDEN_HOST_MARKERS): + return False + if "github" in host and "model" in host: + return False + lowered_model = (model or "").lower() + if any(marker in lowered_model for marker in FORBIDDEN_MODEL_MARKERS): + return False + if (credential_name or "") in FORBIDDEN_CREDENTIAL_NAMES: + return False + return True + + +class ProviderResponseError(RuntimeError): + """Raised when an upstream returns a non-parseable chat.completion body.""" + def is_transient_error(exc: BaseException) -> bool: """Return True when a provider call failure is worth retrying with backoff.""" @@ -299,11 +335,22 @@ def _send(self, agent: ModelAgent, payload: dict[str, Any]) -> str: method="POST", ) with self._open_provider(request) as response: - data = json.loads(response.read().decode("utf-8")) - usage = data.get("usage") - if isinstance(usage, dict): - self._local.usage = usage - return data["choices"][0]["message"]["content"] + raw = response.read().decode("utf-8") + try: + data = json.loads(raw) + usage = data.get("usage") + if isinstance(usage, dict): + self._local.usage = usage + content = data["choices"][0]["message"]["content"] + except (json.JSONDecodeError, KeyError, IndexError, TypeError, AttributeError) as exc: + raise ProviderResponseError( + f"provider {agent.id} returned a malformed chat completion" + ) from exc + if not isinstance(content, str): + raise ProviderResponseError( + f"provider {agent.id} returned a malformed chat completion" + ) + return content def _open_provider(self, request: urllib.request.Request) -> Any: """Open a provider request built from a validated provider URL.""" @@ -1529,13 +1576,29 @@ def _ranked_agents(self, text: str, role: str) -> list[ModelAgent]: lowered = text.lower() return sorted(self.agents, key=lambda agent: self._score_agent(agent, role, lowered), reverse=True) + def _agent_ready(self, agent: ModelAgent) -> bool: + """True when the agent is enabled and (if remote) has a resolvable KV credential.""" + if agent.disabled: + return False + if agent.base_url.startswith("mock://"): + return True + return get_credential(agent.credential_name) is not None + def _select_agent(self, text: str, role: str) -> ModelAgent: - selected = self._ranked_agents(text, role)[0] - if selected.disabled: # pragma: no cover + ready = [ + agent + for agent in self._ranked_agents(text, role) + if self._agent_ready(agent) and role not in agent.provider_exclusions + ] + if not ready: + remote = [agent for agent in self.agents if not agent.base_url.startswith("mock://")] + if remote and not any(self._agent_ready(agent) for agent in remote): + raise NotConfigured( + "no configured provider credential is resolvable; refuse to route " + "(no GitHub Models fallback)" + ) raise RuntimeError(f"no enabled agent available for role={role}") - if role in selected.provider_exclusions: # pragma: no cover - raise RuntimeError(f"no eligible agent available for role={role}") - return selected + return ready[0] def _invoke( self, primary: ModelAgent, messages: list[ChatMessage], *, text: str, role: str @@ -1564,10 +1627,19 @@ def _invoke( def _failover_candidates(self, primary: ModelAgent, text: str, role: str) -> list[ModelAgent]: ranked = self._ranked_agents(text, role) ordered = [primary] + [agent for agent in ranked if agent.id != primary.id] - eligible = [agent for agent in ordered if not agent.disabled and role not in agent.provider_exclusions] + eligible = [ + agent + for agent in ordered + if self._agent_ready(agent) and role not in agent.provider_exclusions + ] + if not eligible: + raise NotConfigured( + "no configured provider credential is resolvable; refuse to route " + "(no GitHub Models fallback)" + ) healthy = [agent for agent in eligible if not self._circuit_open(agent.id)] # If every eligible agent is circuit-open, still probe them rather than fail with no attempt. - return healthy or eligible or [primary] + return healthy or eligible def _circuit_open(self, agent_id: str) -> bool: state = self._circuit.get(agent_id) diff --git a/contextual_orchestrator/provider_catalog.py b/contextual_orchestrator/provider_catalog.py new file mode 100644 index 000000000..eb951a7a2 --- /dev/null +++ b/contextual_orchestrator/provider_catalog.py @@ -0,0 +1,271 @@ +"""Production provider catalog: org secrets, static seed, and optional /v1/models discovery. + +The catalog is data (``examples/agents.production.json``) plus a bootstrap +adapter. Runtime provider keys still resolve only through ``get_credential``. +Environment variables are bootstrap transport into the KV — the same seam as +``register-credential --from-env`` (see ``docs/kv-credentials.md``). + +A missing secret skips that upstream and keeps the rest of the pool serving. +GitHub Models / Copilot tokens are rejected. Providers that expose +``GET /v1/models`` can auto-register chat models; providers without a list API +keep the paper-justified static seed (``docs/doctoring/provider-catalog.md``). +""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import Any +from urllib.parse import urlparse +from urllib.request import Request, urlopen + +from .credentials import get_credential, register_credential +from .orchestrator import ( + FORBIDDEN_CREDENTIAL_NAMES, + FORBIDDEN_HOST_MARKERS, + FORBIDDEN_MODEL_MARKERS, + ModelAgent, + _AgentPoolStore, + catalog_allows_fields, + load_agents, +) + +ORG_CREDENTIAL_NAMES: tuple[str, ...] = ( + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "OPENAI_API_KEY", + "OPENROUTER_API_KEY", + "BYTEZ_API_KEY", +) + +NIM_INTEGRATE_BASE_URL = "https://integrate.api.nvidia.com/v1" +OPENAI_API_BASE_URL = "https://api.openai.com/v1" +OPENROUTER_API_BASE_URL = "https://openrouter.ai/api/v1" +BYTEZ_OPENAI_BASE_URL = "https://api.bytez.com/models/v2/openai/v1" + +PRODUCTION_SEED_PATH = Path(__file__).resolve().parents[1] / "examples" / "agents.production.json" + +_NON_CHAT_MODEL_MARKERS = ( + "embedding", + "whisper", + "tts", + "dall-e", + "dalle", + "moderation", + "transcri", + "tts-1", +) +_DISCOVERY_CAP = 16 + + +def catalog_allows_agent(agent_or_mapping: ModelAgent | dict[str, Any]) -> bool: + """Return True when a seed row or ``ModelAgent`` is not a GitHub Models target.""" + if isinstance(agent_or_mapping, dict): + credential_name = str( + agent_or_mapping.get("api_key_env") or agent_or_mapping.get("credential_key") or "" + ) + return catalog_allows_fields( + str(agent_or_mapping.get("base_url", "")), + str(agent_or_mapping.get("model", "")), + credential_name, + ) + return catalog_allows_fields( + agent_or_mapping.base_url, + agent_or_mapping.model, + agent_or_mapping.credential_name, + ) + + +def load_production_seed(path: str | Path | None = None) -> list[ModelAgent]: + """Load the default production agent catalog (not the mock-only example).""" + return load_agents(str(path or PRODUCTION_SEED_PATH)) + + +def register_org_credentials_from_env(*, skip_missing: bool = True) -> dict[str, list[str]]: + """Register the five org Actions secrets from env into the KV (bootstrap only). + + Missing names are skipped when ``skip_missing`` is true so a partial secret + set still yields a serving pool. This is the single allowed ``os.environ`` + read of provider key *values* — deploy/CI injects them into this one-shot + process; request-time resolution stays on ``get_credential``. + """ + registered: list[str] = [] + skipped: list[str] = [] + for name in ORG_CREDENTIAL_NAMES: + value = os.environ.get(name) + if value: + register_credential(name, value) + registered.append(name) + else: + skipped.append(name) + if not skip_missing: + raise RuntimeError(f"{name} is not set for bootstrap transport") + return {"registered": registered, "skipped": skipped} + + +def parse_models_list(payload: Any) -> list[str]: + """Extract chat model ids from an OpenAI-shaped ``/v1/models`` payload. + + Malformed bodies return an empty list (static seed remains the claim + boundary). Embedding/audio/image ids and retired GitHub Models names are + dropped. + """ + if not isinstance(payload, dict): + return [] + rows = payload.get("data") + if not isinstance(rows, list): + return [] + models: list[str] = [] + seen: set[str] = set() + for row in rows: + if not isinstance(row, dict): + continue + model_id = row.get("id") + if not isinstance(model_id, str) or not model_id.strip(): + continue + lowered = model_id.lower() + if any(marker in lowered for marker in _NON_CHAT_MODEL_MARKERS): + continue + if any(marker in lowered for marker in FORBIDDEN_MODEL_MARKERS): + continue + if model_id in seen: + continue + seen.add(model_id) + models.append(model_id) + return models + + +def discover_provider_models( + base_url: str, + credential_name: str, + *, + allow_insecure: bool = False, + timeout: float = 10.0, +) -> list[str]: + """GET ``{base_url}/models`` with the KV credential; return chat model ids. + + Any transport, HTTP, or parse failure returns ``[]`` so the static seed + stays in force. ``allow_insecure`` is a lab/test hook for loopback fixtures. + """ + api_key = get_credential(credential_name) + if not api_key: + return [] + parsed = urlparse(base_url) + if not parsed.hostname: + return [] + if not allow_insecure and parsed.scheme != "https": + return [] + request = Request( + f"{base_url.rstrip('/')}/models", + headers={"authorization": f"Bearer {api_key}", "accept": "application/json"}, + method="GET", + ) + try: + with urlopen(request, timeout=timeout) as response: # nosec B310 - caller supplies a catalog base_url already used for chat. + payload = json.loads(response.read().decode("utf-8")) + except Exception: # noqa: BLE001 - discovery must never break bootstrap + return [] + return parse_models_list(payload)[:_DISCOVERY_CAP] + + +def _discovered_agent_id(provider_name: str, model: str, existing: set[str]) -> str: + """Build a two-or-more-word snake_case id for a discovered chat model.""" + provider = re.sub(r"[^a-z0-9]+", "_", (provider_name or "discovered").lower()).strip("_") or "discovered" + slug = re.sub(r"[^a-z0-9]+", "_", model.lower()).strip("_") or "model" + base = f"{provider}_{slug}" + if len(base) > 80: + base = base[:80].rstrip("_") + candidate = base + suffix = 2 + while candidate in existing: + candidate = f"{base}_{suffix}" + suffix += 1 + return candidate + + +def compose_provider_catalog( + seed: list[ModelAgent], + *, + discover: bool = False, + allow_insecure_discovery: bool = False, +) -> tuple[list[ModelAgent], list[dict[str, str]]]: + """Keep agents whose KV credential is present; optionally append discovered chat models.""" + ready: list[ModelAgent] = [] + skipped: list[dict[str, str]] = [] + for agent in seed: + if not catalog_allows_agent(agent): + skipped.append({"id": agent.id, "reason": "forbidden_provider"}) + continue + if agent.base_url.startswith("mock://") or get_credential(agent.credential_name): + ready.append(agent) + else: + skipped.append({"id": agent.id, "reason": "credential_missing"}) + if discover: + seen_models = {(agent.base_url, agent.model) for agent in ready} + existing_ids = {agent.id for agent in ready} + templates: dict[tuple[str, str], ModelAgent] = {} + for agent in ready: + templates.setdefault((agent.base_url, agent.credential_name), agent) + for (base_url, credential_name), template in templates.items(): + insecure = allow_insecure_discovery or urlparse(base_url).scheme == "http" + for model in discover_provider_models( + base_url, credential_name, allow_insecure=insecure + ): + if (base_url, model) in seen_models: + continue + if not catalog_allows_fields(base_url, model, credential_name): + continue + agent_id = _discovered_agent_id(template.provider_name, model, existing_ids) + discovered = ModelAgent( + id=agent_id, + model=model, + base_url=base_url, + credential_key=credential_name, + tags=template.tags, + priority=max(0, template.priority - 1), + provider_name=template.provider_name, + ) + ready.append(discovered) + existing_ids.add(agent_id) + seen_models.add((base_url, model)) + return ready, skipped + + +def persist_catalog_to_agents_db(agents: list[ModelAgent], path: str) -> None: + """Write ready agents into the sqlite agent-pool store used by ``--agents-db``.""" + store = _AgentPoolStore(path) + try: + for agent in agents: + store.save(agent) + finally: + store.close() + + +def seed_provider_catalog( + *, + seed_agents: list[ModelAgent] | None = None, + seed_path: str | Path | None = None, + agents_db: str | None = None, + discover: bool = False, + allow_insecure_discovery: bool = False, +) -> dict[str, Any]: + """Register present org secrets, compose the ready pool, and optionally persist it.""" + credentials = register_org_credentials_from_env(skip_missing=True) + agents = seed_agents if seed_agents is not None else load_production_seed(seed_path) + ready, skipped = compose_provider_catalog( + agents, discover=discover, allow_insecure_discovery=allow_insecure_discovery + ) + if agents_db and ready: + persist_catalog_to_agents_db(ready, agents_db) + return { + "registered_credentials": credentials["registered"], + "skipped_credentials": credentials["skipped"], + "ready_agents": [ + {"id": agent.id, "model": agent.model, "credential_key": agent.credential_name} + for agent in ready + ], + "skipped_agents": skipped, + "agents_db": agents_db, + } diff --git a/docs/architecture.md b/docs/architecture.md index c0f63a81e..22a7166b4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -37,6 +37,9 @@ This repository implements the interface and control plane, not the trained coor - `WorkflowStep.access`: Conductor-style visibility control. - `ModelClient`: OpenAI-compatible HTTP client, with `mock://` for local checks. - `contextual_orchestrator.server`: small `/v1/chat/completions` HTTP server. +- `examples/agents.production.json` + `provider_catalog`: org NIM / OpenAI / + OpenRouter / Bytez seed so OpenCode/Strix call one gateway URL. GitHub Models + are out of catalog. See `docs/doctoring/provider-catalog.md`. The deliberate simplification is the policy. The paper systems learn routing and topology from rewards; this lab uses deterministic keyword scoring so the repo runs without training data, GPUs, or vendor credentials. diff --git a/docs/doctoring/provider-catalog.md b/docs/doctoring/provider-catalog.md new file mode 100644 index 000000000..8b6b01239 --- /dev/null +++ b/docs/doctoring/provider-catalog.md @@ -0,0 +1,77 @@ +# Provider catalog doctoring (APA 7) + +This note states what the production agent catalog claims, what it does not +claim, and which papers justify routing, failover, and static-versus-discovered +model lists. Citations follow the *Publication Manual of the American +Psychological Association* (7th ed.). + +## Claim boundary + +The production seed in `examples/agents.production.json` is a **paper-justified +static catalog** of OpenAI-compatible chat workers. It is not a live snapshot of +every model a vendor sells. + +| Claim | Boundary | +| --- | --- | +| NVIDIA NIM primary/secondary | Host is `https://integrate.api.nvidia.com/v1`. Seeded chat models are Llama-3.3-Nemotron-Super-49B-v1.5 and Nemotron-3-Super-120B-A12B, taken from NVIDIA model cards (NVIDIA, 2025, 2026). Additional NIM chat models are appended only when `GET /v1/models` succeeds for `NVIDIA_NIM_API_KEY` or `NVIDIA_NIM_API_KEY_SUB`. | +| OpenAI | Host is `https://api.openai.com/v1`. Static seed uses `gpt-5.5` (already the repo's OpenAI example). Live listing is preferred when `OPENAI_API_KEY` can call `/v1/models`. | +| OpenRouter | Host is `https://openrouter.ai/api/v1`. Static `anthropic/claude-sonnet-4` and `openai/gpt-4.1` are capability tags for coding/review and reasoning until `/v1/models` returns the caller's available set. | +| Bytez | Official OpenAI-compatible base URL is `https://api.bytez.com/models/v2/openai/v1` (Bytez, n.d.). The static chat seed is `Qwen/Qwen3-4B` from that document. A public `/models` list is **not guaranteed**; discovery is best-effort and an empty list keeps this static seed. | +| GitHub Models | **Out of catalog.** `models.github.ai`, Copilot tokens, `gpt-5.6-luna`, and `gpt-5.6-terra` are rejected at agent construction. There is no fallback to GitHub Models when every org secret is missing. | + +Missing a secret disables that upstream only (`NotConfigured` per agent). The +gateway keeps serving every worker whose credential is present. When no +credential is resolvable, routing fail-closes with `NotConfigured` — it does +not invent a GitHub Models worker. + +## Why a multi-upstream catalog (routing papers) + +Cost-optimal cascades and quality-aware routers need **more than one capable +upstream**, then a policy that picks a cheap/fast path or a deeper path +(Chen et al., 2023; Ong et al., 2024; Ding et al., 2024). This repo already +implements Fugu-style `route` versus `conduct` (Sakana AI, 2026), TRINITY +thinker/worker/verifier roles (Zhang et al., 2025), and Conductor access lists +(Li et al., 2025). The production seed only supplies tagged workers those +policies can compose. + +Full-jitter retry on 429/5xx plus cross-agent failover is the operational +reading of a cascade: a rate-limited or malformed upstream must yield to the +next capability-matched worker instead of taking down the single public API +(Chen et al., 2023; Ding et al., 2024). + +## References + +Bytez. (n.d.). *Chat completions*. Bytez Model API. +https://docs.bytez.com/http-reference/examples/openai-compliant/chatCompletionsExample + +Chen, L., Zaharia, M., & Zou, J. (2023). *FrugalGPT: How to use large language +models while reducing cost and improving performance*. arXiv. +https://doi.org/10.48550/arXiv.2305.05176 + +Ding, D., Mallick, A., Wang, C., Sim, R., Mukherjee, S., Rühle, V., Lakshmanan, +L. V. S., & Hassan Awadallah, A. (2024). *Hybrid LLM: Cost-efficient and +quality-aware query routing*. In *Proceedings of the Twelfth International +Conference on Learning Representations*. https://doi.org/10.48550/arXiv.2404.14618 + +Li, Y., et al. (2025). *Learning to orchestrate agents in natural language with +the Conductor*. arXiv. https://doi.org/10.48550/arXiv.2512.04388 + +NVIDIA. (2025). *Llama-3.3-Nemotron-Super-49B-v1.5* [Model card]. +https://build.nvidia.com/nvidia/llama-3_3-nemotron-super-49b-v1_5 + +NVIDIA. (2026). *Nemotron-3-Super-120B-A12B* [Model card]. +https://build.nvidia.com/nvidia/nemotron-3-super-120b-a12b + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., +Kadous, M. W., & Stoica, I. (2024). *RouteLLM: Learning to route LLMs with +preference data*. arXiv. https://doi.org/10.48550/arXiv.2406.18665 + +Sakana AI. (2026, June 22). *Sakana Fugu: One model to command them all*. +https://sakana.ai/fugu-release/ + +Zhang, et al. (2025). *TRINITY: An evolved LLM coordinator*. arXiv. +https://doi.org/10.48550/arXiv.2512.04695 + +Redistributable arXiv PDFs already vendored under `docs/papers/` (FrugalGPT, +RouteLLM, Hybrid LLM) remain the cost/routing evidence pack. Vendor model cards +and the Fugu launch article are cited by URL only; they are not copied here. diff --git a/docs/fuzzing.md b/docs/fuzzing.md index 9897b2bd2..0c7ddbb90 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -31,6 +31,9 @@ deserialize request config validate untrusted input"`): 4. **End-to-end orchestration** — `orchestrator.TaskOrchestrator.run` against `mock://` providers (fully offline). Arbitrary prompt text and mode must produce a JSON-serialisable record whose SSE framing round-trips. +5. **Provider `/v1/models` list** — `provider_catalog.parse_models_list`. + Arbitrary decoded JSON must return a list of non-empty chat model ids and + never raise (static seed remains the claim boundary on junk). ## Running locally diff --git a/docs/kv-credentials.md b/docs/kv-credentials.md index 6860aeeec..abe0cb6e0 100644 --- a/docs/kv-credentials.md +++ b/docs/kv-credentials.md @@ -131,8 +131,27 @@ environment: --name OPENAI_API_KEY --value-stdin ``` -The application test workflow must **not** receive `OPENAI_API_KEY`: tests run on -the mock pool and the in-memory backend and stay green without any secret. +Register all five org Actions secrets in one shot (skip any that are unset): + +```bash +python -m contextual_orchestrator seed-provider-catalog \ + --from-env --skip-missing \ + --agents examples/agents.production.json \ + --agents-db /var/lib/contextual-orchestrator/agents.db \ + --discover-models +``` + +Names: `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, `OPENAI_API_KEY`, +`OPENROUTER_API_KEY`, `BYTEZ_API_KEY`. A missing secret skips that upstream +and keeps the rest of the pool serving. + +The in-memory KV does not survive process exit. CI therefore seeds **inside** +the serve process (`--seed-from-env`); postgres deploys run +`seed-provider-catalog` first. See `docs/opencode-sidecar.md`. + +The application test workflow must **not** receive these provider secrets: +tests run on the mock pool and the in-memory backend and stay green without +any secret. The OpenCode sidecar workflow is the only job that may see them. ## Why this supersedes `api_key_env` diff --git a/docs/library_research.md b/docs/library_research.md index 42c7fa95c..761a3a2b0 100644 --- a/docs/library_research.md +++ b/docs/library_research.md @@ -53,6 +53,16 @@ Extraction triggers: Until those triggers exist, Ponytail recommends strengthening the current single-repo product instead of splitting it. +## Provider catalog + model discovery (2026-08) + +| Area | Library considered | Decision | Evidence | +|---|---|---|---| +| Multi-provider catalog | [LiteLLM](https://github.com/BerriAI/litellm) model list / router | **Do not add LiteLLM.** Keep the seed as JSON data and discover via stdlib `urllib` `GET /v1/models`. | LiteLLM is the product direction, not a current dependency. Ponytail: stdlib HTTP already speaks OpenAI-compatible `/v1/models` (OpenAI, NVIDIA NIM, OpenRouter). Bytez documents chat completions but not a guaranteed list API — static seed is the claim boundary (`docs/doctoring/provider-catalog.md`). Context7/docs: OpenAI Models API; NVIDIA `integrate.api.nvidia.com/v1`; Bytez `https://api.bytez.com/models/v2/openai/v1`. | +| Secret bootstrap | GitHub Actions `secrets.*` + env | **Bootstrap-only env** into `register_credential` / `seed-provider-catalog --from-env`. Runtime stays on `get_credential`. | Matches `docs/kv-credentials.md`. App test job (`tests.yml`) and Security stay secret-free. | +| GitHub Models | GitHub Models inference (`models.github.ai`) | **Rejected.** Org no longer uses GitHub Models. | Fail-closed markers in `catalog_allows_fields`; no `COPILOT_GITHUB_TOKEN`. | + +Skipped: LiteLLM as a runtime dependency, provider SDKs (openai, nvidia-nim), a second catalog store besides `--agents-db` + the KV. + ## Required For New Designs Every new subsystem design must update this file before implementation starts. The entry must name the existing libraries researched, the selected library or stdlib alternative, and the custom code that was deliberately skipped. diff --git a/docs/opencode-sidecar.md b/docs/opencode-sidecar.md new file mode 100644 index 000000000..0e0063247 --- /dev/null +++ b/docs/opencode-sidecar.md @@ -0,0 +1,118 @@ +# OpenCode / Strix sidecar contract + +ContextualWisdomLab no longer uses GitHub Models. OpenCode and Strix should +call **this repo** as one OpenAI-compatible provider. + +``` +register the 5 secrets → serve on localhost → OpenCode provider +baseURL http://127.0.0.1:8000/v1 +model contextual-orchestrator +``` + +App unit tests (`.github/workflows/tests.yml`) and the Security workflow stay +**secret-free**. Seeding lives only in `.github/workflows/opencode-sidecar.yml` +(`workflow_dispatch` / `workflow_call`, not `pull_request`). + +## Tokens + +| Variable | Role | +| --- | --- | +| `CONTEXTUAL_ORCHESTRATOR_TOKEN` | Single local token (`--auth-token`). Enough for OpenCode. | +| `CONTEXTUAL_ORCHESTRATOR_INFERENCE_TOKEN` | Inference-only token when split from `--admin-token`. | +| `CONTEXTUAL_ORCHESTRATOR_ADMIN_TOKEN` | Admin console / `/api/v1/*` when using split tokens. | + +Generate a loopback token: + +```bash +export CONTEXTUAL_ORCHESTRATOR_TOKEN="$(python -c 'import secrets; print(secrets.token_urlsafe(32))')" +``` + +## Org secrets → KV (bootstrap only) + +These GitHub Actions secret *names* are registered into the credential KV. +Runtime still uses `get_credential`, never `os.getenv` at request time. + +| Secret | Typical agents | +| --- | --- | +| `NVIDIA_NIM_API_KEY` | NIM primary Nemotron Super 49B / 120B | +| `NVIDIA_NIM_API_KEY_SUB` | NIM secondary (same host, failover key) | +| `OPENAI_API_KEY` | `https://api.openai.com/v1` | +| `OPENROUTER_API_KEY` | `https://openrouter.ai/api/v1` | +| `BYTEZ_API_KEY` | `https://api.bytez.com/models/v2/openai/v1` | + +A missing secret **skips that upstream** and keeps the rest serving. + +### Same-process CI (in-memory KV) + +Memory KV dies when the process exits, so CI must seed inside the serve process: + +```bash +python -m contextual_orchestrator --serve \ + --seed-from-env \ + --agents examples/agents.production.json \ + --agents-db "$RUNNER_TEMP/agents.db" \ + --host 127.0.0.1 \ + --port 8000 \ + --auth-token "$CONTEXTUAL_ORCHESTRATOR_TOKEN" +``` + +Do not pass --allow-public-bind in CI. Bind stays `127.0.0.1`. + +### Postgres deploy (KV survives process restart) + +```bash +export CONTEXTUAL_ORCHESTRATOR_KV_BACKEND=postgres +export CONTEXTUAL_ORCHESTRATOR_KV_DSN="postgresql://user@host/db" +export CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE="…" + +python -m contextual_orchestrator seed-provider-catalog \ + --from-env --skip-missing \ + --agents examples/agents.production.json \ + --agents-db /var/lib/contextual-orchestrator/agents.db \ + --discover-models + +# or one name at a time +printf '%s' "$OPENAI_API_KEY" | python -m contextual_orchestrator \ + register-credential --name OPENAI_API_KEY --value-stdin +``` + +## OpenCode provider block + +```json +{ + "provider": { + "contextual-orchestrator": { + "npm": "@ai-sdk/openai-compatible", + "name": "Contextual Orchestrator", + "options": { + "baseURL": "http://127.0.0.1:8000/v1", + "apiKey": "{env:CONTEXTUAL_ORCHESTRATOR_TOKEN}" + }, + "models": { + "contextual-orchestrator": { + "name": "contextual-orchestrator" + } + } + } + } +} +``` + +Strix / any OpenAI SDK uses the same `baseURL` and `model`. + +## Smoke curl + +```bash +curl -sS http://127.0.0.1:8000/v1/chat/completions \ + -H "authorization: Bearer $CONTEXTUAL_ORCHESTRATOR_TOKEN" \ + -H "content-type: application/json" \ + -d '{"model":"contextual-orchestrator","messages":[{"role":"user","content":"Write one sentence."}]}' +``` + +Expect HTTP 200 when at least one of the five secrets is registered. When every +secret is missing the gateway fail-closes (`NotConfigured`) and does **not** +fall back to GitHub Models. + +Reusable workflow: `.github/workflows/opencode-sidecar.yml` (`workflow_call`). +The org OpenCode review pipeline in `ContextualWisdomLab/.github` should call +that workflow (or start this server the same way) instead of GitHub Models. diff --git a/docs/papers/README.md b/docs/papers/README.md index 65a89d2af..e9f9b217b 100644 --- a/docs/papers/README.md +++ b/docs/papers/README.md @@ -43,6 +43,12 @@ motivate throughput-oriented **batched** inference and the load-balancing that makes the latency-tolerant batch route economical. Those sources are referenced but not vendored here so this repository remains one deployable control plane. +The 2026-08 production catalog (NIM + OpenAI + OpenRouter + Bytez, no GitHub +Models) reuses these three papers as the routing/cascade evidence pack and +adds Fugu / TRINITY / Conductor (cited in `docs/architecture.md` and +`docs/doctoring/provider-catalog.md`) for how tagged workers are composed. +Vendor model cards are cited by URL only; they are not vendored. + > Citations are provided for scholarly attribution. Redistribution here relies > on the arXiv non-exclusive distribution license each author granted; no > GPL/AGPL-licensed material is vendored anywhere in this repository. diff --git a/examples/agents.production.json b/examples/agents.production.json new file mode 100644 index 000000000..6a70406bf --- /dev/null +++ b/examples/agents.production.json @@ -0,0 +1,85 @@ +{ + "agents": [ + { + "id": "nim_primary_nemotron_super_49b", + "model": "nvidia/llama-3.3-nemotron-super-49b-v1.5", + "base_url": "https://integrate.api.nvidia.com/v1", + "credential_key": "NVIDIA_NIM_API_KEY", + "provider_name": "nvidia_nim", + "tags": ["reasoning", "coding", "planning", "implementation"], + "priority": 10 + }, + { + "id": "nim_primary_nemotron_super_120b", + "model": "nvidia/nemotron-3-super-120b-a12b", + "base_url": "https://integrate.api.nvidia.com/v1", + "credential_key": "NVIDIA_NIM_API_KEY", + "provider_name": "nvidia_nim", + "tags": ["reasoning", "planning", "research", "coding"], + "priority": 9 + }, + { + "id": "nim_secondary_nemotron_super_49b", + "model": "nvidia/llama-3.3-nemotron-super-49b-v1.5", + "base_url": "https://integrate.api.nvidia.com/v1", + "credential_key": "NVIDIA_NIM_API_KEY_SUB", + "provider_name": "nvidia_nim", + "tags": ["reasoning", "coding", "planning"], + "priority": 6 + }, + { + "id": "nim_secondary_nemotron_super_120b", + "model": "nvidia/nemotron-3-super-120b-a12b", + "base_url": "https://integrate.api.nvidia.com/v1", + "credential_key": "NVIDIA_NIM_API_KEY_SUB", + "provider_name": "nvidia_nim", + "tags": ["reasoning", "planning", "research"], + "priority": 5 + }, + { + "id": "openai_primary_general", + "model": "gpt-5.5", + "base_url": "https://api.openai.com/v1", + "credential_key": "OPENAI_API_KEY", + "provider_name": "openai", + "tags": ["reasoning", "writing", "planning", "analysis"], + "priority": 8 + }, + { + "id": "openai_review_worker", + "model": "gpt-5.5", + "base_url": "https://api.openai.com/v1", + "credential_key": "OPENAI_API_KEY", + "provider_name": "openai", + "tags": ["verification", "review", "security"], + "priority": 7 + }, + { + "id": "openrouter_coding_worker", + "model": "anthropic/claude-sonnet-4", + "base_url": "https://openrouter.ai/api/v1", + "credential_key": "OPENROUTER_API_KEY", + "provider_name": "openrouter", + "tags": ["coding", "review", "implementation"], + "priority": 4 + }, + { + "id": "openrouter_reasoning_worker", + "model": "openai/gpt-4.1", + "base_url": "https://openrouter.ai/api/v1", + "credential_key": "OPENROUTER_API_KEY", + "provider_name": "openrouter", + "tags": ["reasoning", "writing", "planning"], + "priority": 3 + }, + { + "id": "bytez_primary_qwen", + "model": "Qwen/Qwen3-4B", + "base_url": "https://api.bytez.com/models/v2/openai/v1", + "credential_key": "BYTEZ_API_KEY", + "provider_name": "bytez", + "tags": ["coding", "writing"], + "priority": 2 + } + ] +} diff --git a/fuzz/targets.py b/fuzz/targets.py index d0c344462..a578ebfbc 100644 --- a/fuzz/targets.py +++ b/fuzz/targets.py @@ -8,7 +8,7 @@ ``AttributeError``, ``RecursionError``, ``SystemError`` or a hang; and * structural invariants on any successful result (shape, types, idempotence). -CodeGraph (``codegraph explore``) surfaced these four surfaces as the ones that +CodeGraph (``codegraph explore``) surfaced these surfaces as the ones that consume untrusted bytes/JSON: 1. ``server._coerce_json`` / ``_validate_mode`` / ``_validate_messages`` / @@ -36,6 +36,7 @@ redact_value, sse_stream_body, ) +from contextual_orchestrator.provider_catalog import parse_models_list # ``RequestError`` is the only *domain* exception the request layer is allowed to # raise; everything else below is a legitimate stdlib decode/parse failure. @@ -107,6 +108,16 @@ def exercise_request_body(raw: bytes) -> None: assert isinstance(message["content"], str) +def exercise_models_list(value: Any) -> None: + """Drive ``parse_models_list`` over arbitrary decoded JSON. + + Invariant: always returns a list of non-empty strings; never raises. + """ + models = parse_models_list(value) + assert isinstance(models, list) + assert all(isinstance(item, str) and item for item in models) + + def exercise_agent_config(value: Any) -> None: """Drive ``ModelAgent.from_dict`` over an arbitrary decoded JSON value.""" if not isinstance(value, dict): diff --git a/tests/fuzz/test_fuzz_properties.py b/tests/fuzz/test_fuzz_properties.py index 7e7b3f347..9de86468f 100644 --- a/tests/fuzz/test_fuzz_properties.py +++ b/tests/fuzz/test_fuzz_properties.py @@ -18,6 +18,7 @@ from fuzz.targets import ( exercise_agent_config, + exercise_models_list, exercise_orchestration, exercise_redaction, exercise_request_body, @@ -73,6 +74,12 @@ def test_request_body_rejects_unhashable_message_role() -> None: exercise_request_body(b'{"messages":[{"role":[],"":[],"modnt":""}]}') +@_SETTINGS +@given(_json_values) +def test_models_list_parser_never_crashes(value: object) -> None: + exercise_models_list(value) + + @_SETTINGS @given(_json_values) def test_agent_config_parser(value: object) -> None: diff --git a/tests/test_catalog_bootstrap.py b/tests/test_catalog_bootstrap.py new file mode 100644 index 000000000..c483dfd92 --- /dev/null +++ b/tests/test_catalog_bootstrap.py @@ -0,0 +1,290 @@ +"""Bootstrap: register org secrets into the KV and compose a ready agent pool. + +Env is bootstrap transport only (docs/kv-credentials.md). A missing secret skips +that upstream and keeps the rest serving — NotConfigured is per-agent, never a +process crash. Providers that expose GET /v1/models are discovered; others keep +the paper-justified static seed (docs/doctoring/provider-catalog.md). +""" + +from __future__ import annotations + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import sys +import tempfile +import threading + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + get_credential, + set_backend, +) +from contextual_orchestrator.provider_catalog import ( # noqa: E402 + ORG_CREDENTIAL_NAMES, + compose_provider_catalog, + discover_provider_models, + parse_models_list, + persist_catalog_to_agents_db, + register_org_credentials_from_env, + seed_provider_catalog, +) + + +@pytest.fixture(autouse=True) +def _fresh_backend(): + set_backend(InMemoryCredentialBackend()) + saved = {name: os.environ.pop(name, None) for name in ORG_CREDENTIAL_NAMES} + try: + yield + finally: + set_backend(None) + for name, value in saved.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def test_register_org_credentials_skips_missing_and_does_not_crash() -> None: + os.environ["OPENAI_API_KEY"] = "sk-openai-present" + os.environ["NVIDIA_NIM_API_KEY"] = "nvapi-present" + report = register_org_credentials_from_env(skip_missing=True) + assert report["registered"] == ["NVIDIA_NIM_API_KEY", "OPENAI_API_KEY"] + assert "BYTEZ_API_KEY" in report["skipped"] + assert "OPENROUTER_API_KEY" in report["skipped"] + assert "NVIDIA_NIM_API_KEY_SUB" in report["skipped"] + assert get_credential("OPENAI_API_KEY") == "sk-openai-present" + assert get_credential("BYTEZ_API_KEY") is None + + +def test_register_org_credentials_can_require_every_name() -> None: + with pytest.raises(RuntimeError, match="NVIDIA_NIM_API_KEY"): + register_org_credentials_from_env(skip_missing=False) + + +def test_discover_provider_models_returns_empty_without_key_or_public_https() -> None: + assert discover_provider_models("https://api.openai.com/v1", "OPENAI_API_KEY") == [] + os.environ["OPENAI_API_KEY"] = "sk-present" + register_org_credentials_from_env(skip_missing=True) + assert discover_provider_models("not-a-url", "OPENAI_API_KEY") == [] + assert discover_provider_models("http://127.0.0.1:9", "OPENAI_API_KEY", allow_insecure=False) == [] + + +def test_discovered_agent_id_truncates_and_avoids_collisions() -> None: + from contextual_orchestrator.provider_catalog import _discovered_agent_id + + long_model = "x" * 120 + short_id = _discovered_agent_id("openai", long_model, set()) + assert len(short_id) <= 80 + first = _discovered_agent_id("openai", "o4-mini", set()) + second = _discovered_agent_id("openai", "o4-mini", {first}) + assert first != second + assert first.startswith("openai_") + + +def test_parse_models_list_dedupes_repeated_chat_ids() -> None: + models = parse_models_list({"data": [{"id": "gpt-5.5"}, {"id": "gpt-5.5"}]}) + assert models == ["gpt-5.5"] + + +def test_register_org_credentials_never_reads_absent_names_as_empty() -> None: + report = register_org_credentials_from_env(skip_missing=True) + assert report["registered"] == [] + assert set(report["skipped"]) == set(ORG_CREDENTIAL_NAMES) + for name in ORG_CREDENTIAL_NAMES: + assert get_credential(name) is None + + +def test_compose_skips_agents_whose_credential_is_missing() -> None: + os.environ["OPENAI_API_KEY"] = "sk-only-openai" + register_org_credentials_from_env(skip_missing=True) + seed = [ + ModelAgent( + "openai_primary_agent", + "gpt-5.5", + "https://api.openai.com/v1", + credential_key="OPENAI_API_KEY", + tags=("reasoning",), + ), + ModelAgent( + "bytez_primary_agent", + "Qwen/Qwen3-4B", + "https://api.bytez.com/models/v2/openai/v1", + credential_key="BYTEZ_API_KEY", + tags=("coding",), + ), + ] + ready, skipped = compose_provider_catalog(seed, discover=False) + assert [agent.id for agent in ready] == ["openai_primary_agent"] + assert [row["id"] for row in skipped] == ["bytez_primary_agent"] + assert skipped[0]["reason"] == "credential_missing" + + +def test_compose_persists_ready_agents_to_agents_db() -> None: + os.environ["OPENAI_API_KEY"] = "sk-persist" + register_org_credentials_from_env(skip_missing=True) + seed = [ + ModelAgent( + "openai_primary_agent", + "gpt-5.5", + "https://api.openai.com/v1", + credential_key="OPENAI_API_KEY", + tags=("reasoning", "writing"), + ) + ] + ready, _skipped = compose_provider_catalog(seed, discover=False) + with tempfile.TemporaryDirectory() as directory: + db_path = os.path.join(directory, "agents.db") + persist_catalog_to_agents_db(ready, db_path) + restarted = TaskOrchestrator( + [ModelAgent("placeholder_agent", "mock-hold", "mock://hold")], + agents_db=db_path, + ) + assert any(agent.id == "openai_primary_agent" for agent in restarted.agents) + assert any(agent.model == "gpt-5.5" for agent in restarted.agents) + + +def test_parse_models_list_keeps_chat_ids_and_drops_non_chat() -> None: + payload = { + "object": "list", + "data": [ + {"id": "gpt-5.5", "object": "model"}, + {"id": "text-embedding-3-large", "object": "model"}, + {"id": "whisper-1", "object": "model"}, + {"id": "dall-e-3", "object": "model"}, + {"id": "tts-1", "object": "model"}, + {"id": "nvidia/llama-3.3-nemotron-super-49b-v1.5"}, + {"id": "gpt-5.6-luna"}, + ], + } + models = parse_models_list(payload) + assert "gpt-5.5" in models + assert "nvidia/llama-3.3-nemotron-super-49b-v1.5" in models + assert "text-embedding-3-large" not in models + assert "whisper-1" not in models + assert "dall-e-3" not in models + assert "tts-1" not in models + assert "gpt-5.6-luna" not in models + + +def test_parse_models_list_is_exception_robust_on_malformed_payloads() -> None: + assert parse_models_list(None) == [] + assert parse_models_list("not-json-object") == [] + assert parse_models_list({"data": "nope"}) == [] + assert parse_models_list({"data": [None, 3, {"id": ""}]}) == [] + + +class _ModelsProvider: + """Serves GET /models with a scripted body (OpenAI list shape).""" + + def __init__(self, status: int, body: object) -> None: + self.status = status + self.body = body + outer = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + raw = json.dumps(outer.body).encode("utf-8") if not isinstance(outer.body, bytes) else outer.body + self.send_response(outer.status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + def log_message(self, *args: object) -> None: + pass + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + def __enter__(self) -> "_ModelsProvider": + self._thread.start() + return self + + def __exit__(self, *exc: object) -> None: + self._server.shutdown() + + @property + def base_url(self) -> str: + return f"http://127.0.0.1:{self._server.server_address[1]}" + + +def test_discover_provider_models_registers_chat_ids_from_list_api() -> None: + os.environ["OPENAI_API_KEY"] = "sk-discover" + register_org_credentials_from_env(skip_missing=True) + listing = {"data": [{"id": "gpt-5.5"}, {"id": "gpt-5.5-mini"}, {"id": "text-embedding-3-small"}]} + with _ModelsProvider(200, listing) as provider: + models = discover_provider_models(provider.base_url, "OPENAI_API_KEY", allow_insecure=True) + assert models == ["gpt-5.5", "gpt-5.5-mini"] + + +def test_discover_provider_models_keeps_static_seed_when_list_api_fails() -> None: + os.environ["OPENAI_API_KEY"] = "sk-discover-fail" + register_org_credentials_from_env(skip_missing=True) + with _ModelsProvider(404, {"error": "no list"}) as provider: + models = discover_provider_models(provider.base_url, "OPENAI_API_KEY", allow_insecure=True) + assert models == [] + with _ModelsProvider(200, b"<< None: + os.environ["OPENAI_API_KEY"] = "sk-seed" + listing = {"data": [{"id": "gpt-5.5"}, {"id": "o4-mini"}]} + with _ModelsProvider(200, listing) as provider: + seed = [ + ModelAgent( + "openai_primary_agent", + "gpt-5.5", + provider.base_url, + credential_key="OPENAI_API_KEY", + tags=("reasoning",), + provider_name="openai", + ), + ModelAgent( + "openrouter_primary_agent", + "anthropic/claude-sonnet-4", + "https://openrouter.ai/api/v1", + credential_key="OPENROUTER_API_KEY", + tags=("review",), + provider_name="openrouter", + ), + ] + with tempfile.TemporaryDirectory() as directory: + db_path = os.path.join(directory, "pool.db") + report = seed_provider_catalog( + seed_agents=seed, + agents_db=db_path, + discover=True, + allow_insecure_discovery=True, + ) + assert "OPENAI_API_KEY" in report["registered_credentials"] + assert "OPENROUTER_API_KEY" in report["skipped_credentials"] + ready_ids = {item["id"] for item in report["ready_agents"]} + assert "openai_primary_agent" in ready_ids + assert "openrouter_primary_agent" not in ready_ids + assert any(item["model"] == "o4-mini" for item in report["ready_agents"]) + assert all("github" not in item["model"].lower() for item in report["ready_agents"]) + + +if __name__ == "__main__": # pragma: no cover + import traceback + + for name, fn in sorted(globals().items()): + if name.startswith("test_") and callable(fn): + try: + fn() + except TypeError: + # pytest fixtures are not available in the script runner + traceback.print_exc() + raise + print(f"ok {name}") + print("ok") diff --git a/tests/test_conventions.py b/tests/test_conventions.py index 644c3f3c3..a7bcfab83 100644 --- a/tests/test_conventions.py +++ b/tests/test_conventions.py @@ -17,11 +17,11 @@ def test_two_word_snake_case_rule() -> None: def test_example_agent_ids_follow_object_name_rule() -> None: - config_path = Path(__file__).resolve().parents[1] / "examples" / "agents.mock.json" - config = json.loads(config_path.read_text(encoding="utf-8")) - - for agent in config["agents"]: - require_object_name(agent["id"], "agent.id") + examples = Path(__file__).resolve().parents[1] / "examples" + for config_path in examples.glob("agents.*.json"): + config = json.loads(config_path.read_text(encoding="utf-8")) + for agent in config["agents"]: + require_object_name(agent["id"], "agent.id") def test_library_research_is_required_design_gate() -> None: diff --git a/tests/test_opencode_sidecar_contract.py b/tests/test_opencode_sidecar_contract.py new file mode 100644 index 000000000..68e19a666 --- /dev/null +++ b/tests/test_opencode_sidecar_contract.py @@ -0,0 +1,88 @@ +"""CI sidecar contract: OpenCode/Strix call this repo as one OpenAI-compatible provider. + +The org no longer uses GitHub Models. ContextualWisdomLab/.github OpenCode review +registers the five Actions secrets into the KV, serves loopback-only, and points +OpenCode at http://127.0.0.1:8000/v1 with model contextual-orchestrator. + +App unit tests (tests.yml) and the Security workflow stay secret-free. +""" + +from __future__ import annotations + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def read_text(relative_path: str) -> str: + return (ROOT / relative_path).read_text(encoding="utf-8") + + +def test_opencode_sidecar_doc_states_exact_env_port_and_smoke_curl() -> None: + text = read_text("docs/opencode-sidecar.md") + for expected in ( + "http://127.0.0.1:8000/v1", + "contextual-orchestrator", + "CONTEXTUAL_ORCHESTRATOR_TOKEN", + "CONTEXTUAL_ORCHESTRATOR_INFERENCE_TOKEN", + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "OPENAI_API_KEY", + "OPENROUTER_API_KEY", + "BYTEZ_API_KEY", + "register-credential", + "seed-provider-catalog", + "--allow-public-bind", + "/v1/chat/completions", + "examples/agents.production.json", + "GitHub Models", + ): + assert expected in text + assert "do not pass --allow-public-bind in ci" in text.lower() + + +def test_sidecar_workflow_is_not_the_app_test_job_and_stays_loopback() -> None: + workflow = read_text(".github/workflows/opencode-sidecar.yml") + tests = read_text(".github/workflows/tests.yml") + security = read_text(".github/workflows/security.yml") + + assert "seed-provider-catalog" in workflow + assert "127.0.0.1" in workflow + assert "--allow-public-bind" not in workflow + assert "workflow_call:" in workflow + assert "workflow_dispatch:" in workflow + assert "secrets.NVIDIA_NIM_API_KEY" in workflow + assert "secrets.OPENAI_API_KEY" in workflow + assert "pull_request:" not in workflow # never inject provider secrets into PR app tests + + for secret_name in ( + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "BYTEZ_API_KEY", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", + ): + assert f"secrets.{secret_name}" not in tests + assert f"secrets.{secret_name}" not in security + + +def test_agents_and_changelog_drop_github_models_as_the_opencode_provider() -> None: + agents = read_text("AGENTS.md") + changelog = read_text("CHANGELOG.md") + doctoring = read_text("docs/doctoring/provider-catalog.md") + assert "GitHub Models" in agents + assert "no longer uses GitHub Models" in agents or "no longer use GitHub Models" in agents + assert "get_credential" in agents + assert "os.environ.get(agent.api_key_env)" not in agents + assert "OpenCode" in changelog + assert "APA" in doctoring + assert "claim boundary" in doctoring.lower() + assert "FrugalGPT" in doctoring + + +if __name__ == "__main__": # pragma: no cover + test_opencode_sidecar_doc_states_exact_env_port_and_smoke_curl() + test_sidecar_workflow_is_not_the_app_test_job_and_stays_loopback() + test_agents_and_changelog_drop_github_models_as_the_opencode_provider() + print("ok") diff --git a/tests/test_provider_catalog.py b/tests/test_provider_catalog.py new file mode 100644 index 000000000..0310c9003 --- /dev/null +++ b/tests/test_provider_catalog.py @@ -0,0 +1,156 @@ +"""Paper-grounded contracts for the production multi-provider agent catalog. + +Fugu (Sakana, 2026) requires a swappable worker pool behind one public API. +TRINITY (arXiv:2512.04695) needs thinker/worker/verifier capability tags. +Conductor (arXiv:2512.04388) needs those workers assignable by role. +FrugalGPT / RouteLLM / Hybrid LLM ground cost-aware multi-upstream selection. + +The production seed is data, not mock-only, and must never include GitHub Models. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.conventions import require_object_name # noqa: E402 +from contextual_orchestrator.provider_catalog import ( # noqa: E402 + BYTEZ_OPENAI_BASE_URL, + FORBIDDEN_CREDENTIAL_NAMES, + FORBIDDEN_HOST_MARKERS, + FORBIDDEN_MODEL_MARKERS, + NIM_INTEGRATE_BASE_URL, + OPENAI_API_BASE_URL, + OPENROUTER_API_BASE_URL, + ORG_CREDENTIAL_NAMES, + PRODUCTION_SEED_PATH, + catalog_allows_agent, + load_production_seed, +) + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_org_credential_names_are_exactly_the_five_actions_secrets() -> None: + assert ORG_CREDENTIAL_NAMES == ( + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "OPENAI_API_KEY", + "OPENROUTER_API_KEY", + "BYTEZ_API_KEY", + ) + assert "COPILOT_GITHUB_TOKEN" not in ORG_CREDENTIAL_NAMES + + +def test_production_seed_file_exists_and_is_not_mock_only() -> None: + assert PRODUCTION_SEED_PATH.is_file() + payload = json.loads(PRODUCTION_SEED_PATH.read_text(encoding="utf-8")) + agents = payload["agents"] + assert len(agents) >= 6 + assert all(not str(item["base_url"]).startswith("mock://") for item in agents) + + +def test_production_seed_covers_required_upstreams_and_nemotron_models() -> None: + agents = load_production_seed() + by_key: dict[str, list[str]] = {} + models = {agent.model for agent in agents} + hosts = {agent.base_url for agent in agents} + for agent in agents: + by_key.setdefault(agent.credential_name, []).append(agent.id) + require_object_name(agent.id, "agent.id") + + assert set(by_key) == set(ORG_CREDENTIAL_NAMES) + assert NIM_INTEGRATE_BASE_URL in hosts + assert OPENAI_API_BASE_URL in hosts + assert OPENROUTER_API_BASE_URL in hosts + assert BYTEZ_OPENAI_BASE_URL in hosts + assert any("nemotron-super-49b" in model or "nemotron-super-49b" in model.replace("_", "-") for model in models) + assert any("120b" in model.lower() and "nemotron" in model.lower() for model in models) + assert any(agent.credential_key == "NVIDIA_NIM_API_KEY_SUB" for agent in agents) + assert any(agent.credential_key == "NVIDIA_NIM_API_KEY" for agent in agents) + + +def test_production_seed_tags_support_route_and_conduct_roles() -> None: + agents = load_production_seed() + all_tags = {tag for agent in agents for tag in agent.tags} + for required in ("coding", "review", "reasoning"): + assert required in all_tags, f"catalog must tag {required} workers for Fugu route vs Conductor/TRINITY conduct" + + +def test_production_seed_rejects_github_models_and_copilot() -> None: + raw = PRODUCTION_SEED_PATH.read_text(encoding="utf-8") + lowered = raw.lower() + for marker in ( + "github.com/models", + "models.github.ai", + "models.inference.ai.azure.com", + "api.githubcopilot.com", + "copilot_github_token", + "gpt-5.6-luna", + "gpt-5.6-terra", + ): + assert marker not in lowered + for agent in load_production_seed(): + assert catalog_allows_agent(agent) is True + + +def test_catalog_allows_agent_rejects_github_models_shapes() -> None: + from contextual_orchestrator import ModelAgent + + forbidden = [ + { + "id": "github_models_agent", + "model": "gpt-4o", + "base_url": "https://models.github.ai/inference", + "credential_key": "OPENAI_API_KEY", + }, + { + "id": "copilot_proxy_agent", + "model": "gpt-5.6-luna", + "base_url": "https://api.openai.com/v1", + "credential_key": "OPENAI_API_KEY", + }, + { + "id": "legacy_copilot_agent", + "model": "gpt-5.5", + "base_url": "https://api.openai.com/v1", + "credential_key": "COPILOT_GITHUB_TOKEN", + }, + ] + for payload in forbidden: + assert catalog_allows_agent(payload) is False + with pytest.raises(ValueError, match="GitHub Models"): + ModelAgent( + "blocked_github_agent", + "gpt-5.6-terra", + "https://models.inference.ai.azure.com/v1", + credential_key="COPILOT_GITHUB_TOKEN", + ) + + +def test_forbidden_markers_are_explicit() -> None: + assert "models.github.ai" in FORBIDDEN_HOST_MARKERS + assert "COPILOT_GITHUB_TOKEN" in FORBIDDEN_CREDENTIAL_NAMES + assert "gpt-5.6-luna" in FORBIDDEN_MODEL_MARKERS + assert "gpt-5.6-terra" in FORBIDDEN_MODEL_MARKERS + + +def test_example_agent_ids_in_every_seed_follow_object_name_rule() -> None: + for path in (ROOT / "examples").glob("agents.*.json"): + payload = json.loads(path.read_text(encoding="utf-8")) + for agent in payload["agents"]: + require_object_name(agent["id"], "agent.id") + + +if __name__ == "__main__": # pragma: no cover + for name, fn in sorted(globals().items()): + if name.startswith("test_") and callable(fn): + fn() + print(f"ok {name}") + print("ok") diff --git a/tests/test_provider_catalog_robustness.py b/tests/test_provider_catalog_robustness.py new file mode 100644 index 000000000..25bc81c10 --- /dev/null +++ b/tests/test_provider_catalog_robustness.py @@ -0,0 +1,284 @@ +"""Exception-robust routing: partial keys, 429 failover, circuit breaker, malformed JSON. + +These are fail-closed contracts, not happy-path demos. FrugalGPT-style cascades +and Hybrid LLM routing only work if a degraded upstream yields to the next +capability-matched worker instead of taking down the gateway (Chen et al., 2023; +Ding et al., 2024). Missing credentials must never fall back to GitHub Models. +""" + +from __future__ import annotations + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import sys +import threading +import urllib.error + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + NotConfigured, + get_credential, + register_credential, + set_backend, +) +from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + + +@pytest.fixture(autouse=True) +def _fresh_backend(): + set_backend(InMemoryCredentialBackend()) + try: + yield + finally: + set_backend(None) + + +def _http_error(code: int) -> urllib.error.HTTPError: + return urllib.error.HTTPError("https://provider.example/chat/completions", code, "err", None, None) + + +class _ScriptedClient(ModelClient): + """Capability-preserving client that scripts per-agent outcomes.""" + + def __init__(self, outcomes: dict[str, list[object]]) -> None: + super().__init__(max_retries=1, retry_backoff=0.0) + self.outcomes = {key: list(value) for key, value in outcomes.items()} + self.calls: list[str] = [] + + def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> str: # type: ignore[override] + self.calls.append(agent.id) + queue = self.outcomes.setdefault(agent.id, []) + if not queue: + return f"[{agent.id}] ok" + item = queue.pop(0) + if isinstance(item, Exception): + raise item + return str(item) + + +def _https_workers() -> list[ModelAgent]: + return [ + ModelAgent( + "primary_nim_agent", + "nvidia/llama-3.3-nemotron-super-49b-v1.5", + "https://integrate.api.nvidia.com/v1", + credential_key="NVIDIA_NIM_API_KEY", + tags=("reasoning", "coding", "writing"), + priority=5, + ), + ModelAgent( + "backup_openai_agent", + "gpt-5.5", + "https://api.openai.com/v1", + credential_key="OPENAI_API_KEY", + tags=("reasoning", "coding", "writing"), + priority=1, + ), + ] + + +def test_one_provider_429_failovers_to_next_capability_matched_agent() -> None: + register_credential("NVIDIA_NIM_API_KEY", "nvapi-test") + register_credential("OPENAI_API_KEY", "sk-test") + client = _ScriptedClient( + { + "primary_nim_agent": [_http_error(429), _http_error(429)], + "backup_openai_agent": ["backup answer"], + } + ) + orchestrator = TaskOrchestrator(_https_workers(), client=client) + result = orchestrator.route_once([{"role": "user", "content": "route this coding task"}]) + assert result["answer"] == "backup answer" + assert result["trace"][0]["served_agent_id"] == "backup_openai_agent" + assert result["trace"][0]["failover_from"] == "primary_nim_agent" + assert client.calls[0] == "primary_nim_agent" + assert "backup_openai_agent" in client.calls + + +def test_one_missing_credential_disables_that_worker_and_others_still_route() -> None: + register_credential("OPENAI_API_KEY", "sk-only") + # NVIDIA_NIM_API_KEY is deliberately absent. + client = _ScriptedClient({"backup_openai_agent": ["openai served"]}) + orchestrator = TaskOrchestrator(_https_workers(), client=client) + result = orchestrator.route_once([{"role": "user", "content": "Write a short status update."}]) + assert result["answer"] == "openai served" + assert "primary_nim_agent" not in client.calls + assert client.calls == ["backup_openai_agent"] + + +def test_all_credentials_missing_fail_closed_without_github_models_fallback() -> None: + client = _ScriptedClient({}) + orchestrator = TaskOrchestrator(_https_workers(), client=client) + with pytest.raises(NotConfigured) as exc: + orchestrator.route_once([{"role": "user", "content": "Write a short status update."}]) + message = str(exc.value).lower() + assert "notconfigured" in type(exc.value).__name__.lower() or "credential" in message or "resolvable" in message + assert "copilot" not in message + assert client.calls == [] + assert all("github" not in agent.base_url for agent in orchestrator.agents) + assert all(agent.credential_name != "COPILOT_GITHUB_TOKEN" for agent in orchestrator.agents) + + +def test_timeout_and_5xx_open_circuit_then_skip_dead_agent() -> None: + register_credential("NVIDIA_NIM_API_KEY", "nvapi-test") + register_credential("OPENAI_API_KEY", "sk-test") + client = _ScriptedClient( + { + "primary_nim_agent": [ + TimeoutError("read timeout"), + _http_error(503), + _http_error(502), + ] + } + ) + orchestrator = TaskOrchestrator(_https_workers(), client=client) + for _ in range(orchestrator.circuit_failure_threshold): + output, served, _usage = orchestrator._invoke( + orchestrator._agent("primary_nim_agent"), + [{"role": "system", "content": "Role: worker"}, {"role": "user", "content": "go"}], + text="go", + role="worker", + ) + assert served == "backup_openai_agent" + assert "backup" in output or served == "backup_openai_agent" + assert orchestrator._circuit_open("primary_nim_agent") is True + candidates = orchestrator._failover_candidates( + orchestrator._agent("primary_nim_agent"), "go", "worker" + ) + assert [agent.id for agent in candidates] == ["backup_openai_agent"] + + +class _FakeChatProvider: + """Scripted OpenAI-compatible /chat/completions over loopback HTTP.""" + + def __init__(self, responses: list[tuple[int, object]]) -> None: + self.request_count = 0 + self.responses = responses + outer = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("content-length", 0)) + self.rfile.read(length) + index = min(outer.request_count, len(outer.responses) - 1) + outer.request_count += 1 + status, body = outer.responses[index] + raw = body if isinstance(body, bytes) else json.dumps(body).encode("utf-8") + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + def log_message(self, *args: object) -> None: + pass + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + def __enter__(self) -> "_FakeChatProvider": + self._thread.start() + return self + + def __exit__(self, *exc: object) -> None: + self._server.shutdown() + + @property + def base_url(self) -> str: + return f"http://127.0.0.1:{self._server.server_address[1]}" + + +class _LoopbackClient(ModelClient): + """Full urllib transport that skips public-HTTPS egress checks for lab fixtures.""" + + def _validate_provider(self, agent: ModelAgent) -> None: + if get_credential(agent.credential_name) is None: + raise NotConfigured( + f"{agent.id} requires a resolvable credential '{agent.credential_name}' in the KV" + ) + + +def test_malformed_provider_response_failovers_instead_of_crashing() -> None: + register_credential("NVIDIA_NIM_API_KEY", "nvapi-test") + register_credential("OPENAI_API_KEY", "sk-test") + good = {"choices": [{"message": {"role": "assistant", "content": "recovered from junk"}}]} + with _FakeChatProvider([(200, {"not": "a completion"})]) as bad, _FakeChatProvider([(200, good)]) as ok: + agents = [ + ModelAgent( + "primary_nim_agent", + "nvidia/llama-3.3-nemotron-super-49b-v1.5", + bad.base_url, + credential_key="NVIDIA_NIM_API_KEY", + tags=("reasoning", "writing"), + priority=5, + ), + ModelAgent( + "backup_openai_agent", + "gpt-5.5", + ok.base_url, + credential_key="OPENAI_API_KEY", + tags=("reasoning", "writing"), + priority=1, + ), + ] + orchestrator = TaskOrchestrator(agents, client=_LoopbackClient(max_retries=0, retry_backoff=0.0)) + result = orchestrator.route_once([{"role": "user", "content": "Write a short status update."}]) + assert result["answer"] == "recovered from junk" + assert result["trace"][0]["served_agent_id"] == "backup_openai_agent" + + +def test_malformed_provider_response_does_not_crash_the_http_gateway() -> None: + register_credential("OPENAI_API_KEY", "sk-test") + with _FakeChatProvider([(200, b"{"), (200, {"choices": []})]) as provider: + agent = ModelAgent( + "solo_openai_agent", + "gpt-5.5", + provider.base_url, + credential_key="OPENAI_API_KEY", + tags=("reasoning", "writing"), + ) + orchestrator = TaskOrchestrator([agent], client=_LoopbackClient(max_retries=0, retry_backoff=0.0)) + token = "sidecar_token" + server = build_server(orchestrator, port=0, security=SecurityConfig(auth_token=token)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + import urllib.request + + request = urllib.request.Request( + f"http://127.0.0.1:{server.server_address[1]}/v1/chat/completions", + data=json.dumps( + { + "model": "contextual-orchestrator", + "messages": [{"role": "user", "content": "Write a short status update."}], + } + ).encode("utf-8"), + headers={ + "authorization": f"Bearer {token}", + "content-type": "application/json", + "connection": "close", + }, + method="POST", + ) + with pytest.raises(urllib.error.HTTPError) as exc: + urllib.request.urlopen(request, timeout=5) + assert exc.value.code in {500, 502, 503} + body = json.loads(exc.value.read().decode("utf-8")) + assert "error" in body + finally: + server.shutdown() + + +if __name__ == "__main__": # pragma: no cover + for name, fn in sorted(globals().items()): + if name.startswith("test_") and callable(fn): + fn() + print(f"ok {name}") + print("ok") From ca2dd9f3ff122e2304c0bfe01698071fbaf9f378 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 16:30:37 +0000 Subject: [PATCH 2/5] feat: choose workers by cost-performance, not list walk Replace keyword scoring and YAML-order failover with a single quality-per-unit-cost chooser. 429/5xx/timeout re-runs that chooser on the remaining healthy pool. Empty pool fail-closes without GitHub Models. Co-authored-by: Seongho Bae --- CHANGELOG.md | 8 +- CLAUDE.md | 2 +- README.md | 3 +- conductor/tracks.md | 2 +- contextual_orchestrator/orchestrator.py | 225 ++++++++++++++++----- docs/architecture.md | 27 ++- docs/doctoring/cost_performance_routing.md | 69 +++++++ docs/doctoring/provider-catalog.md | 9 +- docs/papers/README.md | 6 +- docs/product_planning.md | 5 +- fuzz/targets.py | 2 +- tests/test_cost_performance_chooser.py | 218 ++++++++++++++++++++ tests/test_provider_catalog_robustness.py | 65 +++--- tests/test_provider_reliability.py | 14 +- 14 files changed, 558 insertions(+), 97 deletions(-) create mode 100644 docs/doctoring/cost_performance_routing.md create mode 100644 tests/test_cost_performance_chooser.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e432ef354..6345a038e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,13 @@ ### Changed -- Unconfigured remote workers are skipped at select/failover time. When every +- Fast-path routing is a cost-performance choose (quality per unit operator + cost), not deterministic keyword scoring and not a walk down the seed JSON. + 429 / 5xx / timeout re-runs the same chooser on the remaining healthy pool. + Missing credentials drop that worker from the candidate set. An empty + healthy pool fail-closes (no GitHub Models). Deep `conduct` stays + Conductor-style and still requires a workflow hint. +- Unconfigured remote workers are skipped at select/re-selection time. When every provider credential is missing, routing raises `NotConfigured` and does not fall back to GitHub Models. - Malformed upstream chat.completion bodies raise `ProviderResponseError` so diff --git a/CLAUDE.md b/CLAUDE.md index a965d356c..084bf1326 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,7 +82,7 @@ A stdlib-Python lab implementing a single OpenAI-compatible API that routes, del 2. `TaskOrchestrator.complete()` in `orchestrator.py` picks one of two paths: - **Fast path (`route`)**: select a single worker for simple or latency-sensitive requests. - **Deep path (`conduct`)**: build a natural-language workflow of `thinker → worker → verifier → synthesizer` steps. Each `WorkflowStep` carries an **access list** so a worker sees only the prior outputs deliberately exposed to it. -3. `ModelClient` (infrastructure adapter) executes each step against `mock://` agents (offline, used by tests) or OpenAI-compatible HTTPS providers, with jittered retries for transient errors, failover to the next capability-matched agent, and a per-agent circuit breaker. Provider keys come from the KV via `get_credential`; egress to loopback/private/reserved addresses is blocked. +3. `ModelClient` (infrastructure adapter) executes each step against `mock://` agents (offline, used by tests) or OpenAI-compatible HTTPS providers, with jittered retries for transient errors. Worker selection is a cost-performance choose (quality per unit cost); a 429/5xx/timeout **re-runs that chooser** on the remaining healthy pool rather than walking the seed list. A per-agent circuit breaker excludes a persistently failing provider until it cools down. Provider keys come from the KV via `get_credential`; egress to loopback/private/reserved addresses is blocked. 4. The answer is framed as an OpenAI `chat.completion` (or SSE `chat.completion.chunk` stream). Full orchestration traces are only returned to trusted callers. ### Modules (`contextual_orchestrator/`) diff --git a/README.md b/README.md index 70a020532..56a896fd2 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ One fused orchestration loop: - Deep path: a natural-language workflow is built with planner, worker, verifier, and synthesizer steps. - Each step has an access list, so workers see only the prior outputs intentionally exposed to them. - Agent definitions are data, so provider preference, exclusions, privacy constraints, and mock testing do not require code changes. -- Provider calls are resilient: transient failures (timeouts, 429, 5xx) retry with full-jitter exponential backoff, while caller errors (4xx) fail fast. If an agent still fails, the request fails over to the next capability-matched agent in the pool, and a per-agent circuit breaker skips a persistently failing provider until it cools down. Failover is recorded in the trace (`served_agent_id`, `failover_from`). +- Fast-path selection is a cost-performance choose: one worker from the live pool that maximizes expected quality per unit cost (operator `price_per_million`, TRINITY role tags, measured circuit/latency when present). Seed JSON order and prompt keywords do not pick the winner. Transient failures (timeouts, 429, 5xx) retry with full-jitter exponential backoff on that worker; if it still fails, the **same chooser** runs again on the remaining healthy pool (circuit-open agents excluded). An empty healthy pool fail-closes. Re-selection is recorded in the trace (`served_agent_id`, `failover_from`). See [docs/architecture.md](docs/architecture.md) for the source-backed analysis. @@ -262,6 +262,7 @@ python tests/test_paper_contracts.py python tests/test_provider_catalog.py python tests/test_catalog_bootstrap.py python tests/test_provider_catalog_robustness.py +python tests/test_cost_performance_chooser.py python tests/test_opencode_sidecar_contract.py python tests/test_admin_contract.py python tests/test_conventions.py diff --git a/conductor/tracks.md b/conductor/tracks.md index af49e5170..18874efe5 100644 --- a/conductor/tracks.md +++ b/conductor/tracks.md @@ -4,4 +4,4 @@ |---|---|---| | 001-paper-grounded-orchestrator | active | Implement the source-backed orchestration contract with TDD, DDD, and CDD | | 002-enterprise-design-foundation | active | Add paper-grounded screen design, user stories, REST API, code/DB conventions, and i18n | -| 003-org-provider-catalog | active | Seed NIM/OpenAI/OpenRouter/Bytez into the KV, OpenCode sidecar, exception-robust failover | +| 003-org-provider-catalog | active | Seed NIM/OpenAI/OpenRouter/Bytez into the KV, OpenCode sidecar, cost-performance choose + re-selection | diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 9057350a3..50803d945 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -834,6 +834,9 @@ class TaskOrchestrator: "verifier": ("verification", "security", "review", "debugging"), "synthesizer": ("writing", "reasoning", "planning"), } + # Capability vocabulary only — worker selection does not scan the prompt + # for these keywords. The chooser uses ROLE_TAGS, operator prices, and + # measured circuit/latency signals (see ``_choose_worker``). DOMAIN_HINTS = { "coding": ("code", "bug", "debug", "implement", "repository", "test", "코드", "구현"), "security": ("security", "vulnerability", "xss", "sqli", "auth", "보안"), @@ -1559,23 +1562,6 @@ def _plan(self, task: str) -> list[WorkflowStep]: WorkflowStep(3, "synthesizer", synthesizer, "Produce the final answer, incorporating only verified work.", (0, 1, 2)), ] - def _score_agent(self, agent: ModelAgent, role: str, lowered: str) -> tuple[int, int, str]: - if agent.disabled: - return (-20_000, len(agent.tags), agent.id) - if role in agent.provider_exclusions: - return (-10_000, len(agent.tags), agent.id) - role_score = sum(3 for tag in agent.tags if tag in self.ROLE_TAGS.get(role, ())) - domain_score = 0 - for tag, hints in self.DOMAIN_HINTS.items(): - if tag in agent.tags and any(hint in lowered for hint in hints): - domain_score += 2 - return (role_score + domain_score + agent.priority, len(agent.tags), agent.id) - - def _ranked_agents(self, text: str, role: str) -> list[ModelAgent]: - """Agents sorted best-first for a role; the head is the primary, the tail are failovers.""" - lowered = text.lower() - return sorted(self.agents, key=lambda agent: self._score_agent(agent, role, lowered), reverse=True) - def _agent_ready(self, agent: ModelAgent) -> bool: """True when the agent is enabled and (if remote) has a resolvable KV credential.""" if agent.disabled: @@ -1584,62 +1570,201 @@ def _agent_ready(self, agent: ModelAgent) -> bool: return True return get_credential(agent.credential_name) is not None - def _select_agent(self, text: str, role: str) -> ModelAgent: - ready = [ - agent - for agent in self._ranked_agents(text, role) - if self._agent_ready(agent) and role not in agent.provider_exclusions - ] + def _is_healthy_candidate(self, agent: ModelAgent, role: str, excluded: set[str]) -> bool: + """True when the worker may be chosen: ready, not excluded, circuit closed, role allowed.""" + return ( + agent.id not in excluded + and self._agent_ready(agent) + and not self._circuit_open(agent.id) + and role not in agent.provider_exclusions + ) + + def _healthy_candidates(self, role: str, *, excluded: set[str]) -> list[ModelAgent]: + """Live-pool workers the cost-performance chooser may consider.""" + return [agent for agent in self.agents if self._is_healthy_candidate(agent, role, excluded)] + + def _role_quality(self, agent: ModelAgent, role: str) -> float: + """TRINITY role-tag overlap. Zero means this worker is not tagged for the role.""" + tags = self.ROLE_TAGS.get(role, ()) + return float(sum(1 for tag in tags if tag in agent.tags)) + + def _unit_cost(self, agent: ModelAgent) -> float | None: + """Operator ``price_per_million`` for the model, or None when missing/non-positive. + + Missing and non-positive prices are not treated as free (Chen et al., 2023). + """ + price = self.price_per_million.get(agent.model) + if price is None: + return None + try: + value = float(price) + except (TypeError, ValueError): + return None + if value <= 0: + return None + return value + + def _measured_success(self, agent: ModelAgent) -> float: + """Paper-grounded prior: 1.0 unless the circuit has recorded failures.""" + state = self._circuit.get(agent.id) + failures = float(state["failures"]) if state else 0.0 + return 1.0 / (1.0 + failures) + + def _latency_penalty(self, agent: ModelAgent, *, interactive: bool) -> float: + """Interactive path only: measured trace latency, or 1.0 when none exists. + + No latency is invented. Prior of 1.0 is the documented Hybrid-LLM reading + when the request is not interactive or no samples exist (Ding et al., 2024). + """ + if not interactive: + return 1.0 + samples: list[float] = [] + for run in self._workflow_runs.values(): + for step in run.get("trace") or (): + if step.get("agent_id") != agent.id: + continue + raw = step.get("latency_ms") + if raw in (None, ""): + continue + try: + samples.append(float(raw)) + except (TypeError, ValueError): + continue + if not samples: + return 1.0 + return 1.0 + (sum(samples) / len(samples) / 1000.0) + + def _chooser_key( + self, agent: ModelAgent, *, quality: float, interactive: bool + ) -> tuple[float, float, str, str]: + """Minimize: higher quality-per-cost first, then cheaper, then (model, id).""" + success = self._measured_success(agent) + cost = self._unit_cost(agent) + unit = cost if cost is not None else 1.0 + latency = self._latency_penalty(agent, interactive=interactive) + score = (quality * success) / (unit * latency) + return (-score, unit, agent.model, agent.id) + + def _raise_empty_pool(self, role: str, *, excluded: set[str]) -> None: + """Fail closed: no GitHub Models and no invented worker.""" + remote = [agent for agent in self.agents if not agent.base_url.startswith("mock://")] + ready_remote = [agent for agent in remote if self._agent_ready(agent)] + if remote and not ready_remote: + raise NotConfigured( + "no configured provider credential is resolvable; refuse to route " + "(no GitHub Models fallback)" + ) + if excluded or any(self._circuit_open(agent.id) for agent in self.agents): + raise NotConfigured( + "no healthy worker remains in the live pool; refuse to route " + "(no GitHub Models fallback)" + ) + raise RuntimeError(f"no enabled agent available for role={role}") + + def _selection_pool( + self, role: str, text: str, *, excluded: set[str] + ) -> tuple[list[ModelAgent], dict[str, float], bool]: + """Healthy capable workers, per-agent quality, and interactive flag.""" + ready = self._healthy_candidates(role, excluded=excluded) if not ready: - remote = [agent for agent in self.agents if not agent.base_url.startswith("mock://")] - if remote and not any(self._agent_ready(agent) for agent in remote): - raise NotConfigured( - "no configured provider credential is resolvable; refuse to route " - "(no GitHub Models fallback)" - ) + return [], {}, not self._needs_workflow(text) + qualities = {agent.id: self._role_quality(agent, role) for agent in ready} + if any(value > 0 for value in qualities.values()): + capable = [agent for agent in ready if qualities[agent.id] > 0] + else: + capable = ready + qualities = {agent.id: 1.0 for agent in capable} + priced = [agent for agent in capable if self._unit_cost(agent) is not None] + pool = priced if priced else capable + return pool, qualities, not self._needs_workflow(text) + + def _choose_worker(self, role: str, text: str, *, excluded: set[str] | None = None) -> ModelAgent: + """Pick one worker by quality per unit cost. Seed/JSON order is not a signal. + + Quality is TRINITY role-tag overlap (equal prior of 1.0 when nobody is + tagged). Cost is the operator price table; unpriced workers are excluded + when any priced capable candidate exists, and otherwise share unit cost + 1.0 — prices are never invented (Chen et al., 2023; Ong et al., 2024). + Interactive requests apply a measured-latency penalty only when traces + already recorded ``latency_ms``. + """ + excluded = set(excluded or ()) + pool, qualities, interactive = self._selection_pool(role, text, excluded=excluded) + if not pool: + self._raise_empty_pool(role, excluded=excluded) raise RuntimeError(f"no enabled agent available for role={role}") - return ready[0] + return min( + pool, + key=lambda agent: self._chooser_key( + agent, quality=qualities[agent.id], interactive=interactive + ), + ) + + def _ranked_agents(self, text: str, role: str, *, excluded: set[str] | None = None) -> list[ModelAgent]: + """Healthy workers in chooser order (best first). Not seed-file order.""" + pool, qualities, interactive = self._selection_pool(role, text, excluded=set(excluded or ())) + return sorted( + pool, + key=lambda agent: self._chooser_key( + agent, quality=qualities[agent.id], interactive=interactive + ), + ) + + def _select_agent(self, text: str, role: str) -> ModelAgent: + """Public selection seam: one cost-performance choose, not a list walk.""" + return self._choose_worker(role, text) def _invoke( self, primary: ModelAgent, messages: list[ChatMessage], *, text: str, role: str ) -> tuple[str, str, dict[str, Any] | None]: - """Call the primary agent, failing over across capability-matched agents on error. + """Call the chosen worker; on 429/5xx/timeout re-run the chooser. - Transient retry/backoff happens inside ``ModelClient``; this layer adds - cross-agent failover plus a per-agent circuit breaker, and returns - ``(output, served_agent_id, usage)`` — usage is the provider-reported token - usage when available (else None), so spend analytics can prefer it. + Transient retry/backoff happens inside ``ModelClient``. This layer + re-selects on the remaining healthy pool (circuit-open agents excluded). + That is re-selection, not a fixed YAML/list fallback order. Returns + ``(output, served_agent_id, usage)``. """ - candidates = self._failover_candidates(primary, text, role) + excluded: set[str] = set() last_error: Exception | None = None - for agent in candidates: + first_choice: ModelAgent | None = None + attempted = 0 + while True: + if first_choice is None and self._is_healthy_candidate(primary, role, excluded): + agent = primary + else: + try: + agent = self._choose_worker(role, text, excluded=excluded) + except (NotConfigured, RuntimeError): + if last_error is not None: + raise RuntimeError( + f"all {attempted} candidate agents failed for role={role}" + ) from last_error + raise + if first_choice is None: + first_choice = agent try: output = self.client.chat(agent, messages) - except Exception as exc: # noqa: BLE001 - one agent failing routes to the next + except Exception as exc: # noqa: BLE001 - one failure re-runs the chooser last_error = exc + attempted += 1 self._record_failure(agent.id) + excluded.add(agent.id) continue self._record_success(agent.id) usage = self.client.take_usage() if hasattr(self.client, "take_usage") else None return output, agent.id, usage - raise RuntimeError(f"all {len(candidates)} candidate agents failed for role={role}") from last_error def _failover_candidates(self, primary: ModelAgent, text: str, role: str) -> list[ModelAgent]: + """Remaining healthy pool in chooser order. Circuit-open agents are excluded.""" + del primary # primary is not a list-walk cursor; the chooser ranks the pool ranked = self._ranked_agents(text, role) - ordered = [primary] + [agent for agent in ranked if agent.id != primary.id] - eligible = [ - agent - for agent in ordered - if self._agent_ready(agent) and role not in agent.provider_exclusions - ] - if not eligible: + if not ranked: + self._raise_empty_pool(role, excluded=set()) raise NotConfigured( - "no configured provider credential is resolvable; refuse to route " + "no healthy worker remains in the live pool; refuse to route " "(no GitHub Models fallback)" ) - healthy = [agent for agent in eligible if not self._circuit_open(agent.id)] - # If every eligible agent is circuit-open, still probe them rather than fail with no attempt. - return healthy or eligible + return ranked def _circuit_open(self, agent_id: str) -> bool: state = self._circuit.get(agent_id) diff --git a/docs/architecture.md b/docs/architecture.md index 22a7166b4..cba5acd4a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -41,9 +41,30 @@ This repository implements the interface and control plane, not the trained coor OpenRouter / Bytez seed so OpenCode/Strix call one gateway URL. GitHub Models are out of catalog. See `docs/doctoring/provider-catalog.md`. -The deliberate simplification is the policy. The paper systems learn routing and topology from rewards; this lab uses deterministic keyword scoring so the repo runs without training data, GPUs, or vendor credentials. - -Add learned routing only when there is an evaluation set and logs proving the heuristic policy is the bottleneck. +The product policy is a **cost-performance choose**, not keyword scoring and not +a walk down the seed JSON / YAML list. + +- **Fast path (`route_once` / `/v1/chat/completions`).** Fugu-style single-worker + select (Sakana AI, 2026): pick one live worker from the org pool (NIM, + NIM_SUB, OpenAI, OpenRouter, Bytez — never GitHub Models) that maximizes + expected quality per unit cost. Quality is TRINITY role-tag overlap (Zhang et + al., 2025). Cost is the operator `price_per_million` table already used by + spend analytics; missing or non-positive prices are not treated as free + (Chen et al., 2023; Ong et al., 2024). Interactive requests apply a + measured-latency penalty only when traces already recorded `latency_ms` + (Ding et al., 2024). Seed order, `priority`, and prompt-keyword hits are not + selection signals. +- **Exceptions are re-selection.** On 429 / 5xx / timeout the same chooser + runs again on the remaining healthy pool (circuit-open agents excluded). A + worker with no resolvable KV credential is not a candidate. An empty healthy + pool fail-closes — no GitHub Models, no `COPILOT_GITHUB_TOKEN`. +- **Deep path (`conduct`).** Conductor-style decompose / verify / synthesize + (Li et al., 2025) only when `_needs_workflow` says the task needs it. A + review is not expanded into a multi-agent walk just to burn the pool. + +A trained coordinator may replace this deterministic objective only when an +evaluation set proves it is the bottleneck. That is future work, not the +current product policy. See [doctoring/cost_performance_routing.md](doctoring/cost_performance_routing.md). ## Product Planning Interpretation diff --git a/docs/doctoring/cost_performance_routing.md b/docs/doctoring/cost_performance_routing.md new file mode 100644 index 000000000..3a126beb2 --- /dev/null +++ b/docs/doctoring/cost_performance_routing.md @@ -0,0 +1,69 @@ +# Cost-performance routing (APA 7) + +This note states the **product selection policy** for +`route_once` / `/v1/chat/completions`. It replaces the former deterministic +keyword-scoring description in `docs/architecture.md`. Citations follow the +*Publication Manual of the American Psychological Association* (7th ed.). + +## Claim boundary + +| Claim | Boundary | +| --- | --- | +| One worker per fast-path request | Fugu selects a single worker for the low-latency path (Sakana AI, 2026). The gateway does not walk the seed JSON to “try the next name.” | +| Objective | Maximize expected quality per unit cost. Interactive requests also penalize measured latency when `latency_ms` already exists on traces (Ding et al., 2024). | +| Quality signal | TRINITY role-tag overlap (`ROLE_TAGS`: thinker / worker / verifier / synthesizer) (Zhang et al., 2025). When no candidate has role-tag overlap, every remaining healthy worker shares a documented prior of 1.0. Prompt keywords (`DOMAIN_HINTS`) are **not** a selection signal. | +| Cost signal | Operator `price_per_million` (the same table spend analytics uses). Missing or non-positive prices are **not** treated as free (Chen et al., 2023; Ong et al., 2024). If any priced capable worker exists, unpriced equivalents are excluded. If every capable worker is unpriced, unit cost is the documented prior 1.0 (quality-only). Prices are never invented. | +| Measured extras | Circuit-breaker failures scale success as `1 / (1 + failures)`. Latency penalty applies only on the interactive path and only from recorded traces. No eval-set coordinator is trained here. | +| Deep path | Conductor workflows (decompose / verify / synthesize with access lists) run only when the task needs them (Li et al., 2025). A review is not expanded into a multi-agent walk just to burn the pool. | +| Exceptions | 429 / 5xx / timeout: re-run the **same chooser** on the remaining healthy pool. Circuit-open agents are excluded. A worker with no resolvable KV credential is not a candidate. An empty healthy pool fail-closes. | +| Out of catalog | GitHub Models, `COPILOT_GITHUB_TOKEN`, `models.github.ai`, `gpt-5.6-luna`, and `gpt-5.6-terra` are never candidates. | + +Tie-breaks are `(model, id)` after quality-per-cost. List index, seed order, and +`priority` do not decide the winner. + +## Why this is not a list walk + +FrugalGPT and Hybrid LLM motivate a **cheap capable path first**, then a +stronger path only when needed (Chen et al., 2023; Ding et al., 2024). +RouteLLM frames the same decision as a router over a pool, not a static +fallback order (Ong et al., 2024). Walking `agents[i+1]` after a 429 would +make seed-file order the policy. Re-selection keeps the objective when the +first choice is unhealthy. + +## Priors when a signal is missing + +| Missing signal | Prior | Fail-closed alternative | +| --- | --- | --- | +| No `price_per_million` for a model | Exclude that worker if any priced capable peer exists; else unit cost 1.0 | Do not invent a vendor list price | +| No role-tag overlap in the whole healthy pool | Quality 1.0 for every remaining healthy worker | — | +| No recorded `latency_ms` | Penalty 1.0 (no invented latency) | — | +| No eval-set quality | Role tags + circuit success only | Do not train a coordinator from empty logs | +| Empty healthy pool | — | `NotConfigured` / refuse; no GitHub Models | + +## References + +Chen, L., Zaharia, M., & Zou, J. (2023). *FrugalGPT: How to use large language +models while reducing cost and improving performance*. arXiv. +https://doi.org/10.48550/arXiv.2305.05176 + +Ding, D., Mallick, A., Wang, C., Sim, R., Mukherjee, S., Rühle, V., Lakshmanan, +L. V. S., & Hassan Awadallah, A. (2024). *Hybrid LLM: Cost-efficient and +quality-aware query routing*. In *Proceedings of the Twelfth International +Conference on Learning Representations*. https://doi.org/10.48550/arXiv.2404.14618 + +Li, Y., et al. (2025). *Learning to orchestrate agents in natural language with +the Conductor*. arXiv. https://doi.org/10.48550/arXiv.2512.04388 + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., +Kadous, M. W., & Stoica, I. (2024). *RouteLLM: Learning to route LLMs with +preference data*. arXiv. https://doi.org/10.48550/arXiv.2406.18665 + +Sakana AI. (2026, June 22). *Sakana Fugu: One model to command them all*. +https://sakana.ai/fugu-release/ + +Zhang, et al. (2025). *TRINITY: An evolved LLM coordinator*. arXiv. +https://doi.org/10.48550/arXiv.2512.04695 + +Redistributable arXiv PDFs already vendored under `docs/papers/` (FrugalGPT, +RouteLLM, Hybrid LLM) remain the cost/routing evidence pack. Fugu / TRINITY / +Conductor are cited by URL; they are not copied here. diff --git a/docs/doctoring/provider-catalog.md b/docs/doctoring/provider-catalog.md index 8b6b01239..68022c791 100644 --- a/docs/doctoring/provider-catalog.md +++ b/docs/doctoring/provider-catalog.md @@ -34,10 +34,11 @@ thinker/worker/verifier roles (Zhang et al., 2025), and Conductor access lists (Li et al., 2025). The production seed only supplies tagged workers those policies can compose. -Full-jitter retry on 429/5xx plus cross-agent failover is the operational -reading of a cascade: a rate-limited or malformed upstream must yield to the -next capability-matched worker instead of taking down the single public API -(Chen et al., 2023; Ding et al., 2024). +Full-jitter retry on 429/5xx stays inside one worker. If that worker still +fails, the gateway **re-runs the cost-performance chooser** on the remaining +healthy pool (circuit-open agents excluded). That is re-selection, not “the +next name in the seed file” (Chen et al., 2023; Ding et al., 2024). See +[cost_performance_routing.md](cost_performance_routing.md). ## References diff --git a/docs/papers/README.md b/docs/papers/README.md index e9f9b217b..dfc5c33d4 100644 --- a/docs/papers/README.md +++ b/docs/papers/README.md @@ -45,8 +45,10 @@ but not vendored here so this repository remains one deployable control plane. The 2026-08 production catalog (NIM + OpenAI + OpenRouter + Bytez, no GitHub Models) reuses these three papers as the routing/cascade evidence pack and -adds Fugu / TRINITY / Conductor (cited in `docs/architecture.md` and -`docs/doctoring/provider-catalog.md`) for how tagged workers are composed. +adds Fugu / TRINITY / Conductor (cited in `docs/architecture.md`, +`docs/doctoring/provider-catalog.md`, and +`docs/doctoring/cost_performance_routing.md`) for how tagged workers are +chosen (quality per unit cost) and when a Conductor workflow is required. Vendor model cards are cited by URL only; they are not vendored. > Citations are provided for scholarly attribution. Redistribution here relies diff --git a/docs/product_planning.md b/docs/product_planning.md index 74a4aece7..666217f64 100644 --- a/docs/product_planning.md +++ b/docs/product_planning.md @@ -53,7 +53,10 @@ Enterprise teams want the benefit of collective model intelligence without makin ## Deliberate Non-goals For This Repository -- No learned coordinator training. Keep deterministic routing until there is an evaluation set proving it is the bottleneck. +- No learned coordinator training. Routing is a deterministic cost-performance + objective (quality per unit cost; re-selection on the remaining healthy pool), + not keyword scoring or a YAML list walk. A trained coordinator can replace + this objective only when an evaluation set proves it is the bottleneck. - No visual workflow builder. Tables and trace details are enough until operators need to author complex topologies. - No recursive topology UI. Conductor recursion is a future scaling knob, not an MVP control. - No billing, SSO, or RBAC implementation in the stdlib lab. Document the need; add it with the enterprise stack. diff --git a/fuzz/targets.py b/fuzz/targets.py index a578ebfbc..559b57350 100644 --- a/fuzz/targets.py +++ b/fuzz/targets.py @@ -172,7 +172,7 @@ def _mock_orchestrator() -> TaskOrchestrator: def exercise_orchestration(prompt: str, mode: str) -> None: """Run a full orchestration on arbitrary prompt text against mock providers. - Exercises ``_latest_user_text`` -> ``_needs_workflow`` -> ``_score_agent`` -> + Exercises ``_latest_user_text`` -> ``_needs_workflow`` -> ``_choose_worker`` -> route/conduct -> trace assembly -> SSE framing, all offline via ``mock://``. """ orchestrator = _mock_orchestrator() diff --git a/tests/test_cost_performance_chooser.py b/tests/test_cost_performance_chooser.py new file mode 100644 index 000000000..efd6f5266 --- /dev/null +++ b/tests/test_cost_performance_chooser.py @@ -0,0 +1,218 @@ +"""Cost-performance worker selection: one chooser, not a YAML/list walk. + +Fugu (Sakana AI, 2026) selects a single worker for the low-latency path. +FrugalGPT / RouteLLM / Hybrid LLM maximize quality per unit cost; unpriced +models are not given invented prices (Chen et al., 2023; Ong et al., 2024; +Ding et al., 2024). TRINITY role tags gate capability. Conductor workflows +are out of scope here — ``route_once`` picks one worker. + +Exceptions re-run the same chooser on the remaining healthy pool. Seed JSON +order and prompt keywords must not decide the winner. +""" + +from __future__ import annotations + +from pathlib import Path +import sys +import urllib.error + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + NotConfigured, + register_credential, + set_backend, +) +from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 + + +@pytest.fixture(autouse=True) +def _fresh_backend(): + set_backend(InMemoryCredentialBackend()) + try: + yield + finally: + set_backend(None) + + +class _ScriptedClient(ModelClient): + """Records call order and scripts per-agent outcomes.""" + + def __init__(self, outcomes: dict[str, list[object]] | None = None) -> None: + super().__init__(max_retries=0, retry_backoff=0.0) + self.outcomes = {key: list(value) for key, value in (outcomes or {}).items()} + self.calls: list[str] = [] + + def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> str: # type: ignore[override] + self.calls.append(agent.id) + queue = self.outcomes.setdefault(agent.id, []) + if not queue: + return f"[{agent.id}] ok" + item = queue.pop(0) + if isinstance(item, Exception): + raise item + return str(item) + + +def _http_error(code: int) -> urllib.error.HTTPError: + return urllib.error.HTTPError("https://provider.example/chat/completions", code, "err", None, None) + + +def _equivalent_priced_pool(*, expensive_first: bool) -> list[ModelAgent]: + expensive = ModelAgent( + "expensive_review_agent", + "pricey-review-model", + tags=("reasoning", "writing", "coding"), + ) + cheap = ModelAgent( + "cheap_review_agent", + "thrifty-review-model", + tags=("reasoning", "writing", "coding"), + ) + return [expensive, cheap] if expensive_first else [cheap, expensive] + + +PRICES = {"thrifty-review-model": 1.0, "pricey-review-model": 40.0} + + +def test_cheaper_capable_worker_wins_over_more_expensive_equivalent() -> None: + client = _ScriptedClient() + orchestrator = TaskOrchestrator( + _equivalent_priced_pool(expensive_first=True), + client=client, + price_per_million=PRICES, + ) + result = orchestrator.route_once([{"role": "user", "content": "Write a short status update."}]) + assert result["trace"][0]["agent_id"] == "cheap_review_agent" + assert client.calls == ["cheap_review_agent"] + + +def test_seed_json_order_does_not_determine_the_winner() -> None: + client_a = _ScriptedClient() + client_b = _ScriptedClient() + first = TaskOrchestrator( + _equivalent_priced_pool(expensive_first=True), + client=client_a, + price_per_million=PRICES, + ) + reversed_pool = TaskOrchestrator( + _equivalent_priced_pool(expensive_first=False), + client=client_b, + price_per_million=PRICES, + ) + prompt = [{"role": "user", "content": "Write a short status update."}] + assert first.route_once(prompt)["trace"][0]["agent_id"] == "cheap_review_agent" + assert reversed_pool.route_once(prompt)["trace"][0]["agent_id"] == "cheap_review_agent" + assert client_a.calls == client_b.calls == ["cheap_review_agent"] + + +def test_prompt_keywords_do_not_override_cost_performance() -> None: + # Old keyword scoring would boost a "coding" specialist on this prompt. + # Both workers are equivalently capable; the expensive one must not win. + expensive = ModelAgent( + "keyword_heavy_agent", + "pricey-review-model", + tags=("reasoning", "writing", "coding", "implementation"), + ) + cheap = ModelAgent( + "thrifty_general_agent", + "thrifty-review-model", + tags=("reasoning", "writing", "coding", "implementation"), + ) + orchestrator = TaskOrchestrator( + [expensive, cheap], + client=_ScriptedClient(), + price_per_million=PRICES, + ) + result = orchestrator.route_once( + [{"role": "user", "content": "Please implement and debug this repository test code."}] + ) + assert result["mode"] == "route" + assert result["trace"][0]["agent_id"] == "thrifty_general_agent" + + +def test_429_reselects_with_chooser_not_next_in_file_order() -> None: + # File order is expensive, then cheap. Chooser must pick cheap first. + # After cheap 429s, re-run the chooser on the remaining pool (expensive), + # not "the next name after expensive in the YAML". + client = _ScriptedClient({"cheap_review_agent": [_http_error(429)]}) + orchestrator = TaskOrchestrator( + _equivalent_priced_pool(expensive_first=True), + client=client, + price_per_million=PRICES, + ) + result = orchestrator.route_once([{"role": "user", "content": "Write a short status update."}]) + assert client.calls[0] == "cheap_review_agent" + assert client.calls[1] == "expensive_review_agent" + assert result["trace"][0]["served_agent_id"] == "expensive_review_agent" + assert result["trace"][0]["failover_from"] == "cheap_review_agent" + assert result["answer"] == "[expensive_review_agent] ok" + + +def test_unpriced_equivalent_is_not_given_an_invented_price() -> None: + # Honest prior: when a priced capable worker exists, unpriced workers lose + # (missing price is not treated as free). + priced = ModelAgent("priced_review_agent", "thrifty-review-model", tags=("reasoning", "writing")) + unpriced = ModelAgent("unpriced_review_agent", "mystery-review-model", tags=("reasoning", "writing")) + orchestrator = TaskOrchestrator( + [unpriced, priced], + client=_ScriptedClient(), + price_per_million={"thrifty-review-model": 2.0}, + ) + result = orchestrator.route_once([{"role": "user", "content": "Write a short status update."}]) + assert result["trace"][0]["agent_id"] == "priced_review_agent" + + +def test_missing_credential_is_not_a_candidate() -> None: + register_credential("OPENAI_API_KEY", "sk-only") + missing = ModelAgent( + "nim_missing_agent", + "thrifty-review-model", + "https://integrate.api.nvidia.com/v1", + credential_key="NVIDIA_NIM_API_KEY", + tags=("reasoning", "writing"), + ) + present = ModelAgent( + "openai_ready_agent", + "pricey-review-model", + "https://api.openai.com/v1", + credential_key="OPENAI_API_KEY", + tags=("reasoning", "writing"), + ) + orchestrator = TaskOrchestrator( + [missing, present], + client=_ScriptedClient(), + price_per_million=PRICES, + ) + result = orchestrator.route_once([{"role": "user", "content": "Write a short status update."}]) + assert result["trace"][0]["agent_id"] == "openai_ready_agent" + + +def test_empty_healthy_pool_fail_closes_without_github_models() -> None: + orchestrator = TaskOrchestrator( + [ + ModelAgent( + "remote_openai_agent", + "gpt-5.5", + "https://api.openai.com/v1", + credential_key="OPENAI_API_KEY", + tags=("reasoning",), + ) + ], + client=_ScriptedClient(), + price_per_million={"gpt-5.5": 5.0}, + ) + with pytest.raises(NotConfigured): + orchestrator.route_once([{"role": "user", "content": "Write a short status update."}]) + + +if __name__ == "__main__": # pragma: no cover + for name, fn in sorted(globals().items()): + if name.startswith("test_") and callable(fn): + fn() + print(f"ok {name}") + print("ok") diff --git a/tests/test_provider_catalog_robustness.py b/tests/test_provider_catalog_robustness.py index 25bc81c10..dd08b7aaa 100644 --- a/tests/test_provider_catalog_robustness.py +++ b/tests/test_provider_catalog_robustness.py @@ -1,9 +1,10 @@ """Exception-robust routing: partial keys, 429 failover, circuit breaker, malformed JSON. -These are fail-closed contracts, not happy-path demos. FrugalGPT-style cascades -and Hybrid LLM routing only work if a degraded upstream yields to the next -capability-matched worker instead of taking down the gateway (Chen et al., 2023; -Ding et al., 2024). Missing credentials must never fall back to GitHub Models. +These are fail-closed contracts, not happy-path demos. FrugalGPT / Hybrid LLM +routing re-runs the cost-performance chooser on the remaining healthy pool +when an upstream 429s or returns junk — not a YAML list walk (Chen et al., +2023; Ding et al., 2024). Missing credentials must never fall back to GitHub +Models. """ from __future__ import annotations @@ -63,23 +64,27 @@ def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> s return str(item) +NIM_MODEL = "nvidia/llama-3.3-nemotron-super-49b-v1.5" +OPENAI_MODEL = "gpt-5.5" +WORKER_PRICES = {NIM_MODEL: 1.0, OPENAI_MODEL: 20.0} + + def _https_workers() -> list[ModelAgent]: + # File order is OpenAI then NIM. The chooser must still pick cheaper NIM first. return [ - ModelAgent( - "primary_nim_agent", - "nvidia/llama-3.3-nemotron-super-49b-v1.5", - "https://integrate.api.nvidia.com/v1", - credential_key="NVIDIA_NIM_API_KEY", - tags=("reasoning", "coding", "writing"), - priority=5, - ), ModelAgent( "backup_openai_agent", - "gpt-5.5", + OPENAI_MODEL, "https://api.openai.com/v1", credential_key="OPENAI_API_KEY", tags=("reasoning", "coding", "writing"), - priority=1, + ), + ModelAgent( + "primary_nim_agent", + NIM_MODEL, + "https://integrate.api.nvidia.com/v1", + credential_key="NVIDIA_NIM_API_KEY", + tags=("reasoning", "coding", "writing"), ), ] @@ -93,7 +98,7 @@ def test_one_provider_429_failovers_to_next_capability_matched_agent() -> None: "backup_openai_agent": ["backup answer"], } ) - orchestrator = TaskOrchestrator(_https_workers(), client=client) + orchestrator = TaskOrchestrator(_https_workers(), client=client, price_per_million=WORKER_PRICES) result = orchestrator.route_once([{"role": "user", "content": "route this coding task"}]) assert result["answer"] == "backup answer" assert result["trace"][0]["served_agent_id"] == "backup_openai_agent" @@ -106,7 +111,7 @@ def test_one_missing_credential_disables_that_worker_and_others_still_route() -> register_credential("OPENAI_API_KEY", "sk-only") # NVIDIA_NIM_API_KEY is deliberately absent. client = _ScriptedClient({"backup_openai_agent": ["openai served"]}) - orchestrator = TaskOrchestrator(_https_workers(), client=client) + orchestrator = TaskOrchestrator(_https_workers(), client=client, price_per_million=WORKER_PRICES) result = orchestrator.route_once([{"role": "user", "content": "Write a short status update."}]) assert result["answer"] == "openai served" assert "primary_nim_agent" not in client.calls @@ -115,7 +120,7 @@ def test_one_missing_credential_disables_that_worker_and_others_still_route() -> def test_all_credentials_missing_fail_closed_without_github_models_fallback() -> None: client = _ScriptedClient({}) - orchestrator = TaskOrchestrator(_https_workers(), client=client) + orchestrator = TaskOrchestrator(_https_workers(), client=client, price_per_million=WORKER_PRICES) with pytest.raises(NotConfigured) as exc: orchestrator.route_once([{"role": "user", "content": "Write a short status update."}]) message = str(exc.value).lower() @@ -138,7 +143,7 @@ def test_timeout_and_5xx_open_circuit_then_skip_dead_agent() -> None: ] } ) - orchestrator = TaskOrchestrator(_https_workers(), client=client) + orchestrator = TaskOrchestrator(_https_workers(), client=client, price_per_million=WORKER_PRICES) for _ in range(orchestrator.circuit_failure_threshold): output, served, _usage = orchestrator._invoke( orchestrator._agent("primary_nim_agent"), @@ -211,24 +216,26 @@ def test_malformed_provider_response_failovers_instead_of_crashing() -> None: good = {"choices": [{"message": {"role": "assistant", "content": "recovered from junk"}}]} with _FakeChatProvider([(200, {"not": "a completion"})]) as bad, _FakeChatProvider([(200, good)]) as ok: agents = [ - ModelAgent( - "primary_nim_agent", - "nvidia/llama-3.3-nemotron-super-49b-v1.5", - bad.base_url, - credential_key="NVIDIA_NIM_API_KEY", - tags=("reasoning", "writing"), - priority=5, - ), ModelAgent( "backup_openai_agent", - "gpt-5.5", + OPENAI_MODEL, ok.base_url, credential_key="OPENAI_API_KEY", tags=("reasoning", "writing"), - priority=1, + ), + ModelAgent( + "primary_nim_agent", + NIM_MODEL, + bad.base_url, + credential_key="NVIDIA_NIM_API_KEY", + tags=("reasoning", "writing"), ), ] - orchestrator = TaskOrchestrator(agents, client=_LoopbackClient(max_retries=0, retry_backoff=0.0)) + orchestrator = TaskOrchestrator( + agents, + client=_LoopbackClient(max_retries=0, retry_backoff=0.0), + price_per_million=WORKER_PRICES, + ) result = orchestrator.route_once([{"role": "user", "content": "Write a short status update."}]) assert result["answer"] == "recovered from junk" assert result["trace"][0]["served_agent_id"] == "backup_openai_agent" diff --git a/tests/test_provider_reliability.py b/tests/test_provider_reliability.py index dd5ead98c..270d39701 100644 --- a/tests/test_provider_reliability.py +++ b/tests/test_provider_reliability.py @@ -98,12 +98,20 @@ def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> s def _two_worker_orchestrator(down_id: str) -> tuple[TaskOrchestrator, _AgentDownClient]: + # File order is backup then primary. Prices, not YAML order, pick the first worker. agents = [ - ModelAgent("primary_worker", "mock", tags=("reasoning", "writing"), priority=5), - ModelAgent("backup_worker", "mock", tags=("reasoning", "writing"), priority=1), + ModelAgent("backup_worker", "beta-route-model", tags=("reasoning", "writing")), + ModelAgent("primary_worker", "alpha-route-model", tags=("reasoning", "writing")), ] client = _AgentDownClient(down_id) - return TaskOrchestrator(agents, client=client), client + return ( + TaskOrchestrator( + agents, + client=client, + price_per_million={"alpha-route-model": 1.0, "beta-route-model": 10.0}, + ), + client, + ) def test_failover_to_backup_agent_when_primary_fails() -> None: From 64b69d02005a279274bf57acf9dd93ab9d68aa1a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 16:40:32 +0000 Subject: [PATCH 3/5] fix(security): share chat egress policy with catalog discovery Discovery now reuses the chat-path host/IP checks, refuses redirects that would carry a KV Bearer token, and treats HTTP as an explicit lab hook. Reseed replaces the sqlite agent pool so stale discovered workers cannot linger. Sidecar CI fails when /healthz never comes up. Empty credentials are not treated as ready. Co-authored-by: Seongho Bae --- .github/workflows/opencode-sidecar.yml | 6 ++ CHANGELOG.md | 5 ++ contextual_orchestrator/orchestrator.py | 86 +++++++++++++++------ contextual_orchestrator/provider_catalog.py | 32 ++++---- docs/doctoring/provider-catalog.md | 1 + docs/library_research.md | 1 + docs/opencode-sidecar.md | 7 +- tests/test_catalog_bootstrap.py | 76 +++++++++++++++--- tests/test_opencode_sidecar_contract.py | 2 + tests/test_provider_catalog_robustness.py | 10 +++ 10 files changed, 176 insertions(+), 50 deletions(-) diff --git a/.github/workflows/opencode-sidecar.yml b/.github/workflows/opencode-sidecar.yml index ebfc9a8ba..0cb5e19fc 100644 --- a/.github/workflows/opencode-sidecar.yml +++ b/.github/workflows/opencode-sidecar.yml @@ -80,12 +80,18 @@ jobs: --auth-token "$CONTEXTUAL_ORCHESTRATOR_TOKEN" & server_pid=$! trap 'kill "$server_pid" 2>/dev/null || true' EXIT + ready=0 for _ in 1 2 3 4 5 6 7 8 9 10; do if curl -sf http://127.0.0.1:8000/healthz >/dev/null; then + ready=1 break fi sleep 1 done + if [ "$ready" -ne 1 ]; then + echo "sidecar did not become healthy" + exit 1 + fi registered="$(python -c 'import json,sys; print(len(json.load(open(sys.argv[1]))["registered_credentials"]))' "$RUNNER_TEMP/seed-report.json")" if [ "$registered" = "0" ]; then echo "no provider secrets in this job; skip live chat smoke (fail-closed, no GitHub Models fallback)" diff --git a/CHANGELOG.md b/CHANGELOG.md index 6345a038e..8d1b74f84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,3 +32,8 @@ fall back to GitHub Models. - Malformed upstream chat.completion bodies raise `ProviderResponseError` so the gateway failovers or returns a JSON error instead of crashing. +- Catalog `GET /v1/models` discovery uses the same public-HTTPS egress policy + as chat (no private/loopback/redirect leak of the KV Bearer token). HTTP is + opt-in via `allow_insecure_discovery`. Reseed replaces the sqlite agent pool + so stale discovered workers cannot linger. Sidecar CI fails when `/healthz` + never comes up. diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 50803d945..867978966 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -507,27 +507,9 @@ def _validate_provider(self, agent: ModelAgent) -> None: f"{agent.id} requires a resolvable credential '{agent.credential_name}' in the KV " "(this replaces the legacy api_key_env environment pattern)" ) - parsed = urlparse(agent.base_url) - if parsed.scheme != "https" or not parsed.hostname: - raise RuntimeError(f"{agent.id} base_url must use https") - allowed_hosts = { - host.strip().lower() - for host in os.environ.get("CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS", "").split(",") - if host.strip() - } - hostname = parsed.hostname.lower() - if allowed_hosts and hostname not in allowed_hosts: - raise RuntimeError(f"{agent.id} provider host is not allowlisted") - for address in socket.getaddrinfo(hostname, parsed.port or 443, type=socket.SOCK_STREAM): - ip_address = ipaddress.ip_address(address[4][0]) - if ( - ip_address.is_private - or ip_address.is_loopback - or ip_address.is_link_local - or ip_address.is_multicast - or ip_address.is_reserved - ): - raise RuntimeError(f"{agent.id} provider resolves to non-public address") + reason = provider_base_url_rejection(agent.base_url) + if reason: + raise RuntimeError(f"{agent.id} {reason}") def _provider_url(self, agent: ModelAgent, path: str) -> str: """Build a provider URL while rejecting urllib-supported local schemes.""" @@ -691,6 +673,50 @@ def _coerce_input_text(value: Any) -> str: return " ".join(parts) +def provider_base_url_rejection(base_url: str, *, allow_insecure: bool = False) -> str | None: + """Return why a provider URL must not receive a KV credential, or None if safe. + + Chat (``ModelClient._validate_provider``) and catalog discovery share this + check so ``GET /models`` cannot leak a Bearer token to a host chat would + refuse. ``allow_insecure`` is the lab hook for loopback HTTP fixtures; it + still rejects non-http(s) schemes and non-loopback resolved addresses. + """ + parsed = urlparse(base_url) + if not parsed.hostname: + return "base_url must use https" + if parsed.scheme == "http": + if not allow_insecure: + return "base_url must use https" + elif parsed.scheme != "https": + return "base_url must use https" + allowed_hosts = { + host.strip().lower() + for host in os.environ.get("CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS", "").split(",") + if host.strip() + } + hostname = parsed.hostname.lower() + if allowed_hosts and hostname not in allowed_hosts: + return "provider host is not allowlisted" + port = parsed.port or (80 if parsed.scheme == "http" else 443) + try: + resolved = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + except OSError: + return "provider resolves to non-public address" + for address in resolved: + ip_address = ipaddress.ip_address(address[4][0]) + if allow_insecure and ip_address.is_loopback: + continue + if ( + ip_address.is_private + or ip_address.is_loopback + or ip_address.is_link_local + or ip_address.is_multicast + or ip_address.is_reserved + ): + return "provider resolves to non-public address" + return None + + def load_agents(path: str) -> list[ModelAgent]: # pragma: no cover """Load model agent definitions from an agents JSON file.""" with open(path, encoding="utf-8") as handle: @@ -736,6 +762,21 @@ def load_all(self) -> list["ModelAgent"]: conn.close() return [ModelAgent.from_dict(json.loads(row[0])) for row in rows] + def replace_all(self, agents: list["ModelAgent"]) -> None: + """Replace the stored pool with ``agents`` so a reseed cannot keep stale ids.""" + with self._lock: + conn = sqlite3.connect(self._path) + try: + conn.execute("DELETE FROM agent_pool") + for agent in agents: + conn.execute( + "INSERT INTO agent_pool (agent_id, payload) VALUES (?, ?)", + (agent.id, json.dumps(agent.to_config(), ensure_ascii=False)), + ) + conn.commit() + finally: + conn.close() + def close(self) -> None: """Compatibility no-op: agent-pool operations use short-lived sqlite handles.""" @@ -1568,7 +1609,8 @@ def _agent_ready(self, agent: ModelAgent) -> bool: return False if agent.base_url.startswith("mock://"): return True - return get_credential(agent.credential_name) is not None + secret = get_credential(agent.credential_name) + return bool(secret and str(secret).strip()) def _is_healthy_candidate(self, agent: ModelAgent, role: str, excluded: set[str]) -> bool: """True when the worker may be chosen: ready, not excluded, circuit closed, role allowed.""" diff --git a/contextual_orchestrator/provider_catalog.py b/contextual_orchestrator/provider_catalog.py index eb951a7a2..11e0891a6 100644 --- a/contextual_orchestrator/provider_catalog.py +++ b/contextual_orchestrator/provider_catalog.py @@ -18,8 +18,8 @@ import re from pathlib import Path from typing import Any -from urllib.parse import urlparse -from urllib.request import Request, urlopen +from urllib.error import HTTPError, URLError +from urllib.request import HTTPRedirectHandler, Request, build_opener from .credentials import get_credential, register_credential from .orchestrator import ( @@ -30,6 +30,7 @@ _AgentPoolStore, catalog_allows_fields, load_agents, + provider_base_url_rejection, ) ORG_CREDENTIAL_NAMES: tuple[str, ...] = ( @@ -60,6 +61,13 @@ _DISCOVERY_CAP = 16 +class _RefuseRedirectHandler(HTTPRedirectHandler): + """Refuse redirects so a Bearer token cannot follow a host change.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): # type: ignore[override] + raise HTTPError(req.full_url, code, "redirect refused for credentialed discovery", headers, fp) + + def catalog_allows_agent(agent_or_mapping: ModelAgent | dict[str, Any]) -> bool: """Return True when a seed row or ``ModelAgent`` is not a GitHub Models target.""" if isinstance(agent_or_mapping, dict): @@ -150,21 +158,21 @@ def discover_provider_models( stays in force. ``allow_insecure`` is a lab/test hook for loopback fixtures. """ api_key = get_credential(credential_name) - if not api_key: + if not api_key or not str(api_key).strip(): return [] - parsed = urlparse(base_url) - if not parsed.hostname: - return [] - if not allow_insecure and parsed.scheme != "https": + if provider_base_url_rejection(base_url, allow_insecure=allow_insecure): return [] request = Request( f"{base_url.rstrip('/')}/models", headers={"authorization": f"Bearer {api_key}", "accept": "application/json"}, method="GET", ) + opener = build_opener(_RefuseRedirectHandler) try: - with urlopen(request, timeout=timeout) as response: # nosec B310 - caller supplies a catalog base_url already used for chat. + with opener.open(request, timeout=timeout) as response: payload = json.loads(response.read().decode("utf-8")) + except (HTTPError, URLError, TimeoutError, OSError, json.JSONDecodeError, UnicodeDecodeError): + return [] except Exception: # noqa: BLE001 - discovery must never break bootstrap return [] return parse_models_list(payload)[:_DISCOVERY_CAP] @@ -209,9 +217,8 @@ def compose_provider_catalog( for agent in ready: templates.setdefault((agent.base_url, agent.credential_name), agent) for (base_url, credential_name), template in templates.items(): - insecure = allow_insecure_discovery or urlparse(base_url).scheme == "http" for model in discover_provider_models( - base_url, credential_name, allow_insecure=insecure + base_url, credential_name, allow_insecure=allow_insecure_discovery ): if (base_url, model) in seen_models: continue @@ -234,11 +241,10 @@ def compose_provider_catalog( def persist_catalog_to_agents_db(agents: list[ModelAgent], path: str) -> None: - """Write ready agents into the sqlite agent-pool store used by ``--agents-db``.""" + """Replace the sqlite agent-pool with the current ready set (drop stale ids).""" store = _AgentPoolStore(path) try: - for agent in agents: - store.save(agent) + store.replace_all(agents) finally: store.close() diff --git a/docs/doctoring/provider-catalog.md b/docs/doctoring/provider-catalog.md index 68022c791..e5de9eb1b 100644 --- a/docs/doctoring/provider-catalog.md +++ b/docs/doctoring/provider-catalog.md @@ -18,6 +18,7 @@ every model a vendor sells. | OpenRouter | Host is `https://openrouter.ai/api/v1`. Static `anthropic/claude-sonnet-4` and `openai/gpt-4.1` are capability tags for coding/review and reasoning until `/v1/models` returns the caller's available set. | | Bytez | Official OpenAI-compatible base URL is `https://api.bytez.com/models/v2/openai/v1` (Bytez, n.d.). The static chat seed is `Qwen/Qwen3-4B` from that document. A public `/models` list is **not guaranteed**; discovery is best-effort and an empty list keeps this static seed. | | GitHub Models | **Out of catalog.** `models.github.ai`, Copilot tokens, `gpt-5.6-luna`, and `gpt-5.6-terra` are rejected at agent construction. There is no fallback to GitHub Models when every org secret is missing. | +| Discovery egress | `GET {base_url}/models` uses the same public-HTTPS host/IP policy as chat (`provider_base_url_rejection`). Private, loopback, link-local, reserved, and unallowlisted hosts are refused before the KV Bearer token is sent. Redirects are refused. HTTP is opt-in via `allow_insecure_discovery` (loopback lab fixtures only). | Missing a secret disables that upstream only (`NotConfigured` per agent). The gateway keeps serving every worker whose credential is present. When no diff --git a/docs/library_research.md b/docs/library_research.md index 761a3a2b0..e35b0b87a 100644 --- a/docs/library_research.md +++ b/docs/library_research.md @@ -60,6 +60,7 @@ single-repo product instead of splitting it. | Multi-provider catalog | [LiteLLM](https://github.com/BerriAI/litellm) model list / router | **Do not add LiteLLM.** Keep the seed as JSON data and discover via stdlib `urllib` `GET /v1/models`. | LiteLLM is the product direction, not a current dependency. Ponytail: stdlib HTTP already speaks OpenAI-compatible `/v1/models` (OpenAI, NVIDIA NIM, OpenRouter). Bytez documents chat completions but not a guaranteed list API — static seed is the claim boundary (`docs/doctoring/provider-catalog.md`). Context7/docs: OpenAI Models API; NVIDIA `integrate.api.nvidia.com/v1`; Bytez `https://api.bytez.com/models/v2/openai/v1`. | | Secret bootstrap | GitHub Actions `secrets.*` + env | **Bootstrap-only env** into `register_credential` / `seed-provider-catalog --from-env`. Runtime stays on `get_credential`. | Matches `docs/kv-credentials.md`. App test job (`tests.yml`) and Security stay secret-free. | | GitHub Models | GitHub Models inference (`models.github.ai`) | **Rejected.** Org no longer uses GitHub Models. | Fail-closed markers in `catalog_allows_fields`; no `COPILOT_GITHUB_TOKEN`. | +| Discovery egress | Separate HTTP client / redirect-following `urlopen` | **Reuse chat egress.** `provider_base_url_rejection` + no-redirect opener. HTTP only via `allow_insecure_discovery`. | Same private/loopback/reserved `getaddrinfo` policy as `ModelClient._validate_provider`. Redirects must not carry the KV Bearer token to another host. | Skipped: LiteLLM as a runtime dependency, provider SDKs (openai, nvidia-nim), a second catalog store besides `--agents-db` + the KV. diff --git a/docs/opencode-sidecar.md b/docs/opencode-sidecar.md index 0e0063247..351719d9e 100644 --- a/docs/opencode-sidecar.md +++ b/docs/opencode-sidecar.md @@ -109,9 +109,10 @@ curl -sS http://127.0.0.1:8000/v1/chat/completions \ -d '{"model":"contextual-orchestrator","messages":[{"role":"user","content":"Write one sentence."}]}' ``` -Expect HTTP 200 when at least one of the five secrets is registered. When every -secret is missing the gateway fail-closes (`NotConfigured`) and does **not** -fall back to GitHub Models. +Expect HTTP 200 only when an available provider successfully completes the +request. A registered secret that is expired, quota-limited, or unreachable is +not enough. When every secret is missing the gateway fail-closes +(`NotConfigured`) and does **not** fall back to GitHub Models. Reusable workflow: `.github/workflows/opencode-sidecar.yml` (`workflow_call`). The org OpenCode review pipeline in `ContextualWisdomLab/.github` should call diff --git a/tests/test_catalog_bootstrap.py b/tests/test_catalog_bootstrap.py index c483dfd92..d3253e658 100644 --- a/tests/test_catalog_bootstrap.py +++ b/tests/test_catalog_bootstrap.py @@ -187,10 +187,12 @@ class _ModelsProvider: def __init__(self, status: int, body: object) -> None: self.status = status self.body = body + self.request_count = 0 outer = self class Handler(BaseHTTPRequestHandler): def do_GET(self) -> None: # noqa: N802 + outer.request_count += 1 raw = json.dumps(outer.body).encode("utf-8") if not isinstance(outer.body, bytes) else outer.body self.send_response(outer.status) self.send_header("content-type", "application/json") @@ -275,16 +277,66 @@ def test_seed_provider_catalog_discovers_and_skips_partial_keys() -> None: assert all("github" not in item["model"].lower() for item in report["ready_agents"]) +def test_discover_refuses_loopback_and_http_without_insecure_flag() -> None: + os.environ["OPENAI_API_KEY"] = "sk-ssrf" + register_org_credentials_from_env(skip_missing=True) + listing = {"data": [{"id": "gpt-5.5"}]} + with _ModelsProvider(200, listing) as provider: + assert discover_provider_models(provider.base_url, "OPENAI_API_KEY", allow_insecure=False) == [] + assert provider.request_count == 0 + assert discover_provider_models("https://127.0.0.1:9", "OPENAI_API_KEY") == [] + assert discover_provider_models("https://10.0.0.8:443", "OPENAI_API_KEY") == [] + + +def test_compose_does_not_auto_enable_http_discovery() -> None: + os.environ["OPENAI_API_KEY"] = "sk-http" + register_org_credentials_from_env(skip_missing=True) + listing = {"data": [{"id": "should-not-appear"}]} + with _ModelsProvider(200, listing) as provider: + seed = [ + ModelAgent( + "openai_loopback_agent", + "gpt-5.5", + provider.base_url, + credential_key="OPENAI_API_KEY", + tags=("reasoning",), + provider_name="openai", + ) + ] + ready, _skipped = compose_provider_catalog(seed, discover=True, allow_insecure_discovery=False) + assert [agent.id for agent in ready] == ["openai_loopback_agent"] + assert all(agent.model != "should-not-appear" for agent in ready) + + +def test_reseed_removes_stale_discovered_agents_from_agents_db() -> None: + os.environ["OPENAI_API_KEY"] = "sk-reseed" + register_org_credentials_from_env(skip_missing=True) + stale = ModelAgent( + "stale_discovered_agent", + "old-model", + "https://api.openai.com/v1", + credential_key="OPENAI_API_KEY", + tags=("reasoning",), + ) + current = ModelAgent( + "openai_primary_agent", + "gpt-5.5", + "https://api.openai.com/v1", + credential_key="OPENAI_API_KEY", + tags=("reasoning",), + ) + with tempfile.TemporaryDirectory() as directory: + db_path = os.path.join(directory, "agents.db") + persist_catalog_to_agents_db([stale, current], db_path) + persist_catalog_to_agents_db([current], db_path) + restarted = TaskOrchestrator( + [ModelAgent("placeholder_agent", "mock-hold", "mock://hold")], + agents_db=db_path, + ) + ids = {agent.id for agent in restarted.agents} + assert "openai_primary_agent" in ids + assert "stale_discovered_agent" not in ids + + if __name__ == "__main__": # pragma: no cover - import traceback - - for name, fn in sorted(globals().items()): - if name.startswith("test_") and callable(fn): - try: - fn() - except TypeError: - # pytest fixtures are not available in the script runner - traceback.print_exc() - raise - print(f"ok {name}") - print("ok") + raise SystemExit(pytest.main([__file__])) diff --git a/tests/test_opencode_sidecar_contract.py b/tests/test_opencode_sidecar_contract.py index 68e19a666..9b8b1f084 100644 --- a/tests/test_opencode_sidecar_contract.py +++ b/tests/test_opencode_sidecar_contract.py @@ -55,6 +55,8 @@ def test_sidecar_workflow_is_not_the_app_test_job_and_stays_loopback() -> None: assert "secrets.NVIDIA_NIM_API_KEY" in workflow assert "secrets.OPENAI_API_KEY" in workflow assert "pull_request:" not in workflow # never inject provider secrets into PR app tests + assert "sidecar did not become healthy" in workflow + assert "ready=1" in workflow for secret_name in ( "NVIDIA_NIM_API_KEY", diff --git a/tests/test_provider_catalog_robustness.py b/tests/test_provider_catalog_robustness.py index dd08b7aaa..0aaf11176 100644 --- a/tests/test_provider_catalog_robustness.py +++ b/tests/test_provider_catalog_robustness.py @@ -118,6 +118,16 @@ def test_one_missing_credential_disables_that_worker_and_others_still_route() -> assert client.calls == ["backup_openai_agent"] +def test_empty_or_whitespace_credential_is_not_ready() -> None: + register_credential("NVIDIA_NIM_API_KEY", " ") + register_credential("OPENAI_API_KEY", "") + client = _ScriptedClient({}) + orchestrator = TaskOrchestrator(_https_workers(), client=client, price_per_million=WORKER_PRICES) + with pytest.raises(NotConfigured): + orchestrator.route_once([{"role": "user", "content": "Write a short status update."}]) + assert client.calls == [] + + def test_all_credentials_missing_fail_closed_without_github_models_fallback() -> None: client = _ScriptedClient({}) orchestrator = TaskOrchestrator(_https_workers(), client=client, price_per_million=WORKER_PRICES) From d4bab9b9b094e0b3354e15de21a9290b4225c370 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 07:58:43 +0900 Subject: [PATCH 4/5] chore: re-trigger product gates after concurrent cancel From 0b584ec6ccda87f2ebb4e191c89bc1747255ac75 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 08:20:24 +0900 Subject: [PATCH 5/5] fix(security): nosemgrep FP annotations for cost_ledger SQL and TLS opt-out Match tip substrate: bind-only placeholder SQL and audited provider urllib/TLS paths; product Semgrep gate requires these suppressions. --- contextual_orchestrator/cost_ledger.py | 8 ++++---- contextual_orchestrator/orchestrator.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/contextual_orchestrator/cost_ledger.py b/contextual_orchestrator/cost_ledger.py index d3943c5be..8ca3c6dfd 100644 --- a/contextual_orchestrator/cost_ledger.py +++ b/contextual_orchestrator/cost_ledger.py @@ -583,12 +583,12 @@ def _seed_dimension_catalog(self) -> None: ph = self._placeholder() cur = self._conn.cursor() for order, (name, label, _column) in enumerate(ATTRIBUTION_DIMENSION_CATALOG): - cur.execute( + cur.execute( # nosemgrep -- sqlalchemy-execute-raw-query FP: only the DB-API placeholder char is interpolated; the value is bound. f"SELECT 1 FROM cost_attribution_dimensions WHERE dimension_name = {ph}", # nosec B608 - ph is a DB-API placeholder. (name,), ) if cur.fetchone() is None: - cur.execute( + cur.execute( # nosemgrep -- sqlalchemy-execute-raw-query FP: only DB-API placeholder chars are interpolated; values are bound. "INSERT INTO cost_attribution_dimensions " f"(dimension_name, dimension_label, dimension_order) VALUES ({ph}, {ph}, {ph})", # nosec B608 - ph is a DB-API placeholder. (name, label, order), @@ -602,7 +602,7 @@ def append(self, record: UsageRecord) -> None: placeholders = ", ".join(ph for _ in _USAGE_COLUMNS) columns = ", ".join(_USAGE_COLUMNS) cur = self._conn.cursor() - cur.execute( + cur.execute( # nosemgrep -- sqlalchemy-execute-raw-query FP: columns are the fixed _USAGE_COLUMNS constant; values are bound. f"INSERT INTO llm_usage_records ({columns}) VALUES ({placeholders})", # nosec B608 - columns are fixed _USAGE_COLUMNS. tuple(row.get(column) for column in _USAGE_COLUMNS), ) @@ -622,7 +622,7 @@ def query(self, start: Optional[int] = None, end: Optional[int] = None) -> List[ where = f" WHERE {' AND '.join(clauses)}" if clauses else "" columns = ", ".join(_USAGE_COLUMNS) cur = self._conn.cursor() - cur.execute(f"SELECT {columns} FROM llm_usage_records{where}", tuple(params)) # nosec B608 - columns and clauses are fixed. + cur.execute(f"SELECT {columns} FROM llm_usage_records{where}", tuple(params)) # nosec B608 - columns and clauses are fixed. # nosemgrep -- sqlalchemy-execute-raw-query FP: fixed columns and clause templates; all values are bound. return [dict(zip(_USAGE_COLUMNS, values)) for values in cur.fetchall()] diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 867978966..82e40b2f9 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -266,7 +266,7 @@ def __init__( @staticmethod def _build_ssl_context(ca_bundle: str | None, verify_tls: bool) -> ssl.SSLContext: if not verify_tls: - return ssl._create_unverified_context() # nosec B323 - explicit dev-only provider TLS opt-out. + return ssl._create_unverified_context() # nosec B323 - explicit dev-only provider TLS opt-out. # nosemgrep -- unverified-ssl-context: intentional, default-secure (verify_tls defaults True) dev-only opt-out for self-signed endpoints. if ca_bundle: if not os.path.isfile(ca_bundle): raise ValueError(f"provider CA bundle does not exist: {ca_bundle}") @@ -354,7 +354,7 @@ def _send(self, agent: ModelAgent, payload: dict[str, Any]) -> str: def _open_provider(self, request: urllib.request.Request) -> Any: """Open a provider request built from a validated provider URL.""" - return urllib.request.urlopen( # nosec B310 - request URL comes from _provider_url after provider validation. + return urllib.request.urlopen( # nosec B310 - request URL comes from _provider_url after provider validation. # nosemgrep -- dynamic-urllib-use: URL is built by _provider_url after scheme/host validation; egress to loopback/private/reserved is blocked. request, timeout=self.timeout, context=self._ssl_context,