From 2c414891eef81b85c6223d5ac3f0750c6d4b38cf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:39:49 +0900 Subject: [PATCH 01/15] test: define durable provider catalog contracts --- tests/test_provider_catalog.py | 409 +++++++++++++++++++++++++++++++++ 1 file changed, 409 insertions(+) create mode 100644 tests/test_provider_catalog.py diff --git a/tests/test_provider_catalog.py b/tests/test_provider_catalog.py new file mode 100644 index 00000000..43c44e62 --- /dev/null +++ b/tests/test_provider_catalog.py @@ -0,0 +1,409 @@ +"""Contracts for durable multi-provider discovery, bootstrap, and routing.""" + +from __future__ import annotations + +from dataclasses import replace +import json +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + get_credential, + set_backend, +) +from contextual_orchestrator.provider_catalog import ( # noqa: E402 + DEFAULT_PROVIDER_ACCOUNTS, + PROVIDER_CATALOG_SCHEMA_SQL, + CatalogHttpError, + DiscoveredModel, + InMemoryProviderCatalogStore, + ProviderAwareModelClient, + ProviderCatalogHttpClient, + ProviderCatalogService, + ProviderCatalogUnavailable, + bootstrap_provider_credentials, + build_catalog_orchestrator, + normalize_models_document, +) +from contextual_orchestrator.orchestrator import ModelAgent # noqa: E402 + + +@pytest.fixture(autouse=True) +def _isolated_credentials(): + """Keep every provider bootstrap test isolated from ambient credentials.""" + set_backend(InMemoryCredentialBackend()) + try: + yield + finally: + set_backend(None) + + +def _models(*names: str) -> list[DiscoveredModel]: + """Build deterministic model fixtures for one provider account.""" + return [ + DiscoveredModel( + model_name=name, + display_name=name, + capabilities=("chat", "reasoning"), + modalities=("text",), + context_window=131_072, + input_price_usd_per_million=1.0, + output_price_usd_per_million=2.0, + ) + for name in names + ] + + +def test_default_accounts_cover_every_configured_secret_and_split_nvidia_accounts() -> None: + """The built-in catalog maps all five GitHub secret names without collapsing NIM keys.""" + credential_names = [account.credential_name for account in DEFAULT_PROVIDER_ACCOUNTS] + assert credential_names == [ + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "BYTEZ_API_KEY", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", + ] + account_ids = {account.provider_account_id for account in DEFAULT_PROVIDER_ACCOUNTS} + assert account_ids == { + "nvidia_nim_primary", + "nvidia_nim_secondary", + "bytez_primary", + "openrouter_primary", + "openai_primary", + } + + +def test_bootstrap_registers_all_credentials_without_returning_values() -> None: + """One-shot environment transport writes every secret into KV and reports names only.""" + environment = { + account.credential_name: f"secret-{index}-value" + for index, account in enumerate(DEFAULT_PROVIDER_ACCOUNTS) + } + + summary = bootstrap_provider_credentials(environment, require_all=True) + + assert summary == { + "registered_credentials": [account.credential_name for account in DEFAULT_PROVIDER_ACCOUNTS], + "missing_credentials": [], + } + for account in DEFAULT_PROVIDER_ACCOUNTS: + assert get_credential(account.credential_name) == environment[account.credential_name] + assert "secret-" not in json.dumps(summary) + + +def test_bootstrap_fails_closed_before_partial_write_when_required_secret_is_missing() -> None: + """Required bootstrap validates the complete fixed inventory before mutating KV.""" + environment = { + account.credential_name: "configured-value" + for account in DEFAULT_PROVIDER_ACCOUNTS[:-1] + } + + with pytest.raises(ProviderCatalogUnavailable, match="inventory is incomplete"): + bootstrap_provider_credentials(environment, require_all=True) + + assert all(get_credential(account.credential_name) is None for account in DEFAULT_PROVIDER_ACCOUNTS) + + +def test_optional_bootstrap_registers_present_credentials_and_reports_missing_names() -> None: + """Non-production bootstrap may seed a subset while keeping missing names explicit.""" + first, second = DEFAULT_PROVIDER_ACCOUNTS[:2] + summary = bootstrap_provider_credentials( + {first.credential_name: "primary-value"}, + require_all=False, + accounts=(first, second), + ) + assert summary == { + "registered_credentials": [first.credential_name], + "missing_credentials": [second.credential_name], + } + + +def test_models_document_normalizes_openai_shape_and_rejects_invalid_rows() -> None: + """OpenAI-compatible listings become bounded provider-neutral model records.""" + models = normalize_models_document( + { + "data": [ + { + "id": "alpha/reasoner", + "context_length": 200_000, + "pricing": {"prompt": "0.000001", "completion": "0.000002"}, + }, + { + "id": "vision-model", + "architecture": { + "input_modalities": ["text", "image"], + "output_modalities": ["text"], + }, + }, + {"id": ""}, + {"object": "model"}, + 42, + ] + } + ) + + assert [model.model_name for model in models] == ["alpha/reasoner", "vision-model"] + assert models[0].context_window == 200_000 + assert models[0].input_price_usd_per_million == pytest.approx(1.0) + assert models[0].output_price_usd_per_million == pytest.approx(2.0) + assert "reasoning" in models[0].capabilities + assert "vision" in models[1].capabilities + assert models[1].modalities == ("image", "text") + + +def test_models_document_accepts_models_mapping_and_string_rows() -> None: + """Provider-specific mapping and string inventories normalize without adapter branching.""" + models = normalize_models_document({"models": {"first": "plain-model", "second": {"name": "embed-model"}}}) + assert [model.model_name for model in models] == ["embed-model", "plain-model"] + assert models[0].capabilities == ("embeddings",) + assert models[1].capabilities == ("chat",) + + +def test_models_document_rejects_malformed_root_and_unsafe_numeric_metadata() -> None: + """Malformed roots and non-finite/negative metadata never enter routing evidence.""" + assert normalize_models_document({"data": "not-a-list"}) == [] + model = normalize_models_document( + { + "data": [ + { + "id": "safe-model", + "context_length": -1, + "pricing": {"prompt": "nan", "completion": "-1"}, + } + ] + } + )[0] + assert model.context_window is None + assert model.input_price_usd_per_million is None + assert model.output_price_usd_per_million is None + + +def test_http_client_retries_transient_failure_with_bounded_backoff() -> None: + """Transient catalog errors retry, while the successful document is normalized once.""" + sleeps: list[float] = [] + client = ProviderCatalogHttpClient( + max_attempts=2, + sleep=sleeps.append, + random_uniform=lambda _low, high: high, + ) + calls: list[int] = [] + + def fake_request(_account, _credential): + calls.append(1) + if len(calls) == 1: + raise CatalogHttpError("catalog_http_503", transient=True) + return {"data": [{"id": "recovered-model"}]} + + client._request_json = fake_request # type: ignore[method-assign] + models = client.discover(DEFAULT_PROVIDER_ACCOUNTS[0], "credential") + assert [model.model_name for model in models] == ["recovered-model"] + assert len(calls) == 2 + assert sleeps == [0.5] + + +def test_http_client_does_not_retry_permanent_or_empty_catalog() -> None: + """Authentication and structurally empty catalogs fail fast with stable codes.""" + client = ProviderCatalogHttpClient(max_attempts=3, sleep=lambda _delay: None) + client._request_json = lambda _account, _credential: (_ for _ in ()).throw( # type: ignore[method-assign] + CatalogHttpError("catalog_authentication_failed") + ) + with pytest.raises(CatalogHttpError, match="catalog_authentication_failed"): + client.discover(DEFAULT_PROVIDER_ACCOUNTS[0], "credential") + + client._request_json = lambda _account, _credential: {"data": []} # type: ignore[method-assign] + with pytest.raises(CatalogHttpError, match="catalog_contains_no_models"): + client.discover(DEFAULT_PROVIDER_ACCOUNTS[0], "credential") + + +def test_refresh_isolates_provider_failure_and_preserves_last_known_good_catalog() -> None: + """A failed account refresh cannot erase its prior usable model set or stop peers.""" + store = InMemoryProviderCatalogStore() + primary, secondary = DEFAULT_PROVIDER_ACCOUNTS[:2] + store.replace_catalog(primary, _models("nim-primary-old")) + store.replace_catalog(secondary, _models("nim-secondary-old")) + bootstrap_provider_credentials( + { + primary.credential_name: "primary-secret", + secondary.credential_name: "secondary-secret", + }, + require_all=False, + accounts=(primary, secondary), + ) + + def discover(account, _credential): + if account.provider_account_id == primary.provider_account_id: + raise CatalogHttpError("provider_unavailable", transient=True) + return _models("nim-secondary-new") + + service = ProviderCatalogService(store=store, accounts=(primary, secondary), discover=discover) + summary = service.refresh_all() + + assert summary["provider_accounts"][primary.provider_account_id]["status"] == "stale_available" + assert summary["provider_accounts"][secondary.provider_account_id]["status"] == "refreshed" + enabled = {(row.provider_account_id, row.model.model_name) for row in store.enabled_models()} + assert (primary.provider_account_id, "nim-primary-old") in enabled + assert (secondary.provider_account_id, "nim-secondary-new") in enabled + assert (secondary.provider_account_id, "nim-secondary-old") not in enabled + + +def test_refresh_classifies_missing_credentials_disabled_accounts_and_adapter_failures() -> None: + """Account-local configuration and unexpected adapter exceptions remain explicit.""" + first = DEFAULT_PROVIDER_ACCOUNTS[0] + disabled = replace(DEFAULT_PROVIDER_ACCOUNTS[1], enabled=False) + third = DEFAULT_PROVIDER_ACCOUNTS[2] + bootstrap_provider_credentials({third.credential_name: "configured"}, require_all=False, accounts=(third,)) + service = ProviderCatalogService( + store=InMemoryProviderCatalogStore(), + accounts=(first, disabled, third), + discover=lambda _account, _credential: (_ for _ in ()).throw(RuntimeError("private detail")), + ) + with pytest.raises(ProviderCatalogUnavailable): + service.refresh_all() + rows = service.last_refresh_summary["provider_accounts"] + assert rows[first.provider_account_id]["error_code"] == "credential_not_registered" + assert rows[disabled.provider_account_id]["status"] == "disabled" + assert rows[third.provider_account_id]["error_code"] == "catalog_adapter_failure" + assert "private detail" not in json.dumps(rows) + + +def test_refresh_raises_only_when_no_fresh_or_last_known_good_candidate_exists() -> None: + """An empty first bootstrap fails loudly instead of starting with a mock or empty pool.""" + account = DEFAULT_PROVIDER_ACCOUNTS[0] + bootstrap_provider_credentials({account.credential_name: "secret"}, require_all=False, accounts=(account,)) + service = ProviderCatalogService( + store=InMemoryProviderCatalogStore(), + accounts=(account,), + discover=lambda _account, _credential: (_ for _ in ()).throw( + CatalogHttpError("provider_unavailable", transient=True) + ), + ) + with pytest.raises(ProviderCatalogUnavailable, match="no usable provider model"): + service.refresh_all() + + +def test_catalog_builds_distinct_agents_and_keeps_credential_names_not_values() -> None: + """Discovered models become valid ModelAgent rows with provider-account isolation.""" + store = InMemoryProviderCatalogStore() + primary, secondary = DEFAULT_PROVIDER_ACCOUNTS[:2] + store.replace_catalog(primary, _models("shared-model")) + store.replace_catalog(secondary, _models("shared-model")) + + agents = ProviderCatalogService(store=store, accounts=(primary, secondary)).candidate_agents() + + assert len(agents) == 2 + assert len({agent.id for agent in agents}) == 2 + assert {agent.credential_key for agent in agents} == { + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + } + assert {agent.provider_name for agent in agents} == {"nvidia_nim"} + assert all("secret" not in json.dumps(agent.to_config()).lower() for agent in agents) + + +def test_catalog_orchestrator_uses_role_tags_and_retains_cross_provider_failover() -> None: + """The paper-grounded orchestrator receives role-capable candidates from distinct providers.""" + store = InMemoryProviderCatalogStore() + reasoning_account = DEFAULT_PROVIDER_ACCOUNTS[0] + coding_account = DEFAULT_PROVIDER_ACCOUNTS[3] + store.replace_catalog( + reasoning_account, + [DiscoveredModel("deep-reasoner", "Deep Reasoner", ("chat", "reasoning"), ("text",), 200_000)], + ) + store.replace_catalog( + coding_account, + [DiscoveredModel("code-specialist", "Code Specialist", ("chat", "coding"), ("text",), 128_000)], + ) + + orchestrator = build_catalog_orchestrator(store, accounts=(reasoning_account, coding_account)) + + assert len(orchestrator.agents) == 2 + assert orchestrator._select_agent("plan and analyze", "thinker").model == "deep-reasoner" + assert orchestrator._select_agent("implement this code", "worker").model == "code-specialist" + assert len(orchestrator._failover_candidates(orchestrator.agents[0], "verify", "verifier")) == 2 + + +def test_disabled_provider_account_is_excluded_without_deleting_catalog_history() -> None: + """Governance can disable an account while retaining its catalog and refresh evidence.""" + store = InMemoryProviderCatalogStore() + account = DEFAULT_PROVIDER_ACCOUNTS[0] + store.replace_catalog(account, _models("candidate-model")) + disabled = replace(account, enabled=False) + store.upsert_account(disabled) + + assert ProviderCatalogService(store=store, accounts=(disabled,)).candidate_agents() == [] + assert len(store.all_models()) == 1 + + +def test_bytez_client_uses_native_key_transport_and_normalizes_output() -> None: + """Bytez candidates use their native contract instead of a fabricated OpenAI bearer call.""" + captured: list[tuple[ModelAgent, list[dict[str, str]], str]] = [] + + def bytez_request(agent, messages, credential): + captured.append((agent, messages, credential)) + return {"output": {"content": "bytez-answer"}} + + client = ProviderAwareModelClient(bytez_request=bytez_request) + agent = ModelAgent( + "bytez_worker", + "owner/model", + "https://api.bytez.com", + credential_key="BYTEZ_API_KEY", + provider_name="bytez", + ) + bootstrap_provider_credentials({"BYTEZ_API_KEY": "bytez-secret"}, require_all=False, accounts=(DEFAULT_PROVIDER_ACCOUNTS[2],)) + + answer = client.chat(agent, [{"role": "user", "content": "hello"}]) + + assert answer == "bytez-answer" + assert captured[0][2] == "bytez-secret" + assert client.take_usage() is None + + +def test_bytez_client_fails_closed_on_missing_credential_or_unsupported_output() -> None: + """Native transport never sends an empty key or accepts an ambiguous provider result.""" + agent = ModelAgent( + "bytez_worker", + "owner/model", + "https://api.bytez.com", + credential_key="BYTEZ_API_KEY", + provider_name="bytez", + ) + client = ProviderAwareModelClient(bytez_request=lambda _agent, _messages, _credential: {"output": []}) + with pytest.raises(ProviderCatalogUnavailable, match="credential is not registered"): + client.chat(agent, [{"role": "user", "content": "hello"}]) + + bootstrap_provider_credentials({"BYTEZ_API_KEY": "secret"}, require_all=False, accounts=(DEFAULT_PROVIDER_ACCOUNTS[2],)) + with pytest.raises(ProviderCatalogUnavailable, match="response shape is unsupported"): + client.chat(agent, [{"role": "user", "content": "hello"}]) + + +def test_non_bytez_client_delegates_to_existing_model_client_mock_path() -> None: + """Provider awareness leaves the existing mock/OpenAI-compatible behavior unchanged.""" + client = ProviderAwareModelClient() + agent = ModelAgent("general_agent", "mock-generalist", "mock://local") + assert client.chat(agent, [{"role": "user", "content": "hello"}]) + + +def test_schema_is_normalized_and_never_stores_provider_secret_values() -> None: + """The production catalog DDL keeps credentials referenced by name in normalized tables.""" + normalized = " ".join(PROVIDER_CATALOG_SCHEMA_SQL.lower().split()) + for table_name in ( + "provider_accounts", + "provider_models", + "model_capabilities", + "model_modalities", + "catalog_refresh_runs", + ): + assert f"create table if not exists {table_name}" in normalized + assert "references provider_accounts" in normalized + assert "references provider_models" in normalized + assert "credential_name" in normalized + assert "secret_value" not in normalized + assert "api_key_value" not in normalized + assert "encrypted_value" not in normalized From 1558436f55bdbe996a02863a281cdab7d7cfd9a6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:43:26 +0900 Subject: [PATCH 02/15] feat: add durable multi-provider model catalog --- contextual_orchestrator/provider_catalog.py | 1202 +++++++++++++++++++ 1 file changed, 1202 insertions(+) create mode 100644 contextual_orchestrator/provider_catalog.py diff --git a/contextual_orchestrator/provider_catalog.py b/contextual_orchestrator/provider_catalog.py new file mode 100644 index 00000000..5d65d904 --- /dev/null +++ b/contextual_orchestrator/provider_catalog.py @@ -0,0 +1,1202 @@ +"""Durable provider discovery, catalog persistence, and agent-pool construction. + +GitHub Actions or another trusted bootstrap process may transport the fixed +provider credential inventory into :mod:`contextual_orchestrator.credentials`. +Runtime inference resolves credential names from that registry and never reads +provider API keys directly from ambient environment variables. + +Provider metadata is stored separately from secret values in a normalized +catalog. Account refreshes are isolated: a failed refresh preserves that +account's last-known-good models, while a first deployment with no usable model +fails closed instead of silently starting a mock or empty pool. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import hashlib +import http.client +import ipaddress +import json +import math +import os +import random +import re +import socket +import ssl +import sys +import time +from typing import Any, Callable, Iterable, Mapping, Protocol, Sequence +from urllib.parse import quote, urlparse + +from .credentials import get_credential, register_credential +from .orchestrator import ModelAgent, ModelClient, TaskOrchestrator + + +CATALOG_RESPONSE_MAX_BYTES = 8 * 1024 * 1024 +"""Maximum accepted bytes in one provider model-catalog response.""" + +PROVIDER_CATALOG_SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS provider_accounts ( + provider_account_id text PRIMARY KEY, + provider_name text NOT NULL, + credential_name text NOT NULL, + base_url text NOT NULL, + models_path text, + transport_name text NOT NULL, + auth_header_name text NOT NULL, + auth_prefix text NOT NULL, + enabled_flag boolean NOT NULL DEFAULT true, + priority_rank integer NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS provider_models ( + provider_model_id text PRIMARY KEY, + provider_account_id text NOT NULL REFERENCES provider_accounts(provider_account_id), + model_name text NOT NULL, + display_name text NOT NULL, + context_window integer, + input_price_usd_per_million numeric(20, 8), + output_price_usd_per_million numeric(20, 8), + enabled_flag boolean NOT NULL DEFAULT true, + first_discovered_at timestamptz NOT NULL, + last_seen_at timestamptz NOT NULL, + UNIQUE (provider_account_id, model_name) +); + +CREATE TABLE IF NOT EXISTS model_capabilities ( + provider_model_id text NOT NULL REFERENCES provider_models(provider_model_id) ON DELETE CASCADE, + capability_name text NOT NULL, + PRIMARY KEY (provider_model_id, capability_name) +); + +CREATE TABLE IF NOT EXISTS model_modalities ( + provider_model_id text NOT NULL REFERENCES provider_models(provider_model_id) ON DELETE CASCADE, + modality_name text NOT NULL, + PRIMARY KEY (provider_model_id, modality_name) +); + +CREATE TABLE IF NOT EXISTS catalog_refresh_runs ( + catalog_refresh_id text PRIMARY KEY, + provider_account_id text NOT NULL REFERENCES provider_accounts(provider_account_id), + refresh_status text NOT NULL, + observed_model_count integer NOT NULL DEFAULT 0, + error_code text, + started_at timestamptz NOT NULL, + finished_at timestamptz NOT NULL +); + +CREATE INDEX IF NOT EXISTS provider_models_account_idx + ON provider_models (provider_account_id, enabled_flag); +CREATE INDEX IF NOT EXISTS catalog_refresh_account_idx + ON catalog_refresh_runs (provider_account_id, finished_at DESC); +""" +"""Normalized PostgreSQL schema for provider accounts, models, and refresh evidence.""" + + +@dataclass(frozen=True) +class ProviderAccount: + """One independently governed provider account and credential reference.""" + + provider_account_id: str + provider_name: str + credential_name: str + base_url: str + models_path: str | None = "/models" + transport_name: str = "openai_compatible" + auth_header_name: str = "Authorization" + auth_prefix: str = "Bearer" + enabled: bool = True + priority_rank: int = 0 + + @property + def models_url(self) -> str | None: + """Return the complete model-list endpoint, or ``None`` when unsupported.""" + if self.models_path is None: + return None + return f"{self.base_url.rstrip('/')}/{self.models_path.lstrip('/')}" + + +@dataclass(frozen=True) +class DiscoveredModel: + """Provider-neutral metadata for one discovered model identifier.""" + + model_name: str + display_name: str + capabilities: tuple[str, ...] = ("chat",) + modalities: tuple[str, ...] = ("text",) + context_window: int | None = None + input_price_usd_per_million: float | None = None + output_price_usd_per_million: float | None = None + + +@dataclass(frozen=True) +class CatalogModelRecord: + """A discovered model associated with its provider account.""" + + provider_account_id: str + model: DiscoveredModel + + +class ProviderCatalogUnavailable(RuntimeError): + """Raised when a durable catalog cannot produce any usable provider model.""" + + +class CatalogHttpError(RuntimeError): + """Stable, secret-free provider catalog transport failure.""" + + def __init__(self, code: str, *, transient: bool = False) -> None: + super().__init__(code) + self.code = code + self.transient = transient + + +class ProviderCatalogStore(Protocol): + """Persistence contract shared by in-memory tests and PostgreSQL production.""" + + def upsert_account(self, account: ProviderAccount) -> None: + """Insert or update one provider account without storing a secret value.""" + ... + + def replace_catalog(self, account: ProviderAccount, models: Sequence[DiscoveredModel]) -> None: + """Atomically replace one successful provider account's current model set.""" + ... + + def record_failure(self, account: ProviderAccount, error_code: str) -> None: + """Record a failed refresh without changing the last-known-good model set.""" + ... + + def enabled_models(self) -> list[CatalogModelRecord]: + """Return usable models belonging to enabled provider accounts.""" + ... + + def all_models(self) -> list[CatalogModelRecord]: + """Return catalog history including models on disabled accounts.""" + ... + + def has_models(self, provider_account_id: str) -> bool: + """Return whether an account retains any enabled last-known-good model.""" + ... + + +DEFAULT_PROVIDER_ACCOUNTS: tuple[ProviderAccount, ...] = ( + ProviderAccount( + provider_account_id="nvidia_nim_primary", + provider_name="nvidia_nim", + credential_name="NVIDIA_NIM_API_KEY", + base_url="https://integrate.api.nvidia.com/v1", + priority_rank=1, + ), + ProviderAccount( + provider_account_id="nvidia_nim_secondary", + provider_name="nvidia_nim", + credential_name="NVIDIA_NIM_API_KEY_SUB", + base_url="https://integrate.api.nvidia.com/v1", + priority_rank=0, + ), + ProviderAccount( + provider_account_id="bytez_primary", + provider_name="bytez", + credential_name="BYTEZ_API_KEY", + base_url="https://api.bytez.com", + models_path="/models/v2", + transport_name="bytez_v2", + auth_prefix="Key", + priority_rank=0, + ), + ProviderAccount( + provider_account_id="openrouter_primary", + provider_name="openrouter", + credential_name="OPENROUTER_API_KEY", + base_url="https://openrouter.ai/api/v1", + priority_rank=1, + ), + ProviderAccount( + provider_account_id="openai_primary", + provider_name="openai", + credential_name="OPENAI_API_KEY", + base_url="https://api.openai.com/v1", + priority_rank=1, + ), +) +"""Fixed bootstrap inventory corresponding to the five organization secrets.""" + + +class InMemoryProviderCatalogStore: + """Deterministic catalog store for tests and standalone evaluation.""" + + def __init__(self) -> None: + self._accounts: dict[str, ProviderAccount] = {} + self._models: dict[str, dict[str, DiscoveredModel]] = {} + self.refresh_runs: list[dict[str, Any]] = [] + + def upsert_account(self, account: ProviderAccount) -> None: + """Store an account definition, preserving its model history.""" + self._accounts[account.provider_account_id] = account + + def replace_catalog(self, account: ProviderAccount, models: Sequence[DiscoveredModel]) -> None: + """Replace one account catalog and append successful refresh evidence.""" + self.upsert_account(account) + unique = {model.model_name: model for model in models if model.model_name} + self._models[account.provider_account_id] = unique + now = _utc_now_text() + self.refresh_runs.append( + { + "catalog_refresh_id": _refresh_id(account.provider_account_id, now), + "provider_account_id": account.provider_account_id, + "refresh_status": "refreshed", + "observed_model_count": len(unique), + "error_code": None, + "started_at": now, + "finished_at": now, + } + ) + + def record_failure(self, account: ProviderAccount, error_code: str) -> None: + """Append failure evidence while leaving the prior model mapping unchanged.""" + self.upsert_account(account) + now = _utc_now_text() + self.refresh_runs.append( + { + "catalog_refresh_id": _refresh_id(account.provider_account_id, now), + "provider_account_id": account.provider_account_id, + "refresh_status": "failed", + "observed_model_count": 0, + "error_code": error_code, + "started_at": now, + "finished_at": now, + } + ) + + def enabled_models(self) -> list[CatalogModelRecord]: + """Return sorted model rows whose provider account is enabled.""" + records: list[CatalogModelRecord] = [] + for account_id, models in self._models.items(): + if not self._accounts[account_id].enabled: + continue + records.extend(CatalogModelRecord(account_id, model) for model in models.values()) + return sorted(records, key=lambda row: (row.provider_account_id, row.model.model_name)) + + def all_models(self) -> list[CatalogModelRecord]: + """Return every retained model regardless of account enablement.""" + return sorted( + ( + CatalogModelRecord(account_id, model) + for account_id, models in self._models.items() + for model in models.values() + ), + key=lambda row: (row.provider_account_id, row.model.model_name), + ) + + def has_models(self, provider_account_id: str) -> bool: + """Return whether an account has at least one retained model.""" + return bool(self._models.get(provider_account_id)) + + +class PostgresProviderCatalogStore: # pragma: no cover - production database adapter + """Normalized PostgreSQL provider catalog with account-scoped transactions.""" + + def __init__(self, dsn: str) -> None: + if not dsn: + raise ProviderCatalogUnavailable("provider catalog requires a PostgreSQL DSN") + self._dsn = dsn + self._schema_ready = False + + def _connect(self): + try: + import psycopg + except ImportError as exc: + raise ProviderCatalogUnavailable( + "provider catalog requires contextual-orchestrator[db]" + ) from exc + return psycopg.connect(self._dsn) + + def _ensure_schema(self, connection: Any) -> None: + if self._schema_ready: + return + with connection.cursor() as cursor: + cursor.execute(PROVIDER_CATALOG_SCHEMA_SQL) + connection.commit() + self._schema_ready = True + + def upsert_account(self, account: ProviderAccount) -> None: + with self._connect() as connection: + self._ensure_schema(connection) + with connection.cursor() as cursor: + _upsert_account_row(cursor, account) + connection.commit() + + def replace_catalog(self, account: ProviderAccount, models: Sequence[DiscoveredModel]) -> None: + started_at = _utc_now() + unique = {model.model_name: model for model in models if model.model_name} + with self._connect() as connection: + self._ensure_schema(connection) + with connection.cursor() as cursor: + _upsert_account_row(cursor, account) + seen_ids: list[str] = [] + for model in unique.values(): + model_id = _provider_model_id(account.provider_account_id, model.model_name) + seen_ids.append(model_id) + cursor.execute( + "INSERT INTO provider_models (" + "provider_model_id, provider_account_id, model_name, display_name, " + "context_window, input_price_usd_per_million, output_price_usd_per_million, " + "enabled_flag, first_discovered_at, last_seen_at) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, true, %s, %s) " + "ON CONFLICT (provider_model_id) DO UPDATE SET " + "display_name = EXCLUDED.display_name, context_window = EXCLUDED.context_window, " + "input_price_usd_per_million = EXCLUDED.input_price_usd_per_million, " + "output_price_usd_per_million = EXCLUDED.output_price_usd_per_million, " + "enabled_flag = true, last_seen_at = EXCLUDED.last_seen_at", + ( + model_id, + account.provider_account_id, + model.model_name, + model.display_name, + model.context_window, + model.input_price_usd_per_million, + model.output_price_usd_per_million, + started_at, + started_at, + ), + ) + cursor.execute( + "DELETE FROM model_capabilities WHERE provider_model_id = %s", + (model_id,), + ) + cursor.execute( + "DELETE FROM model_modalities WHERE provider_model_id = %s", + (model_id,), + ) + for capability in model.capabilities: + cursor.execute( + "INSERT INTO model_capabilities (provider_model_id, capability_name) " + "VALUES (%s, %s) ON CONFLICT DO NOTHING", + (model_id, capability), + ) + for modality in model.modalities: + cursor.execute( + "INSERT INTO model_modalities (provider_model_id, modality_name) " + "VALUES (%s, %s) ON CONFLICT DO NOTHING", + (model_id, modality), + ) + if seen_ids: + cursor.execute( + "UPDATE provider_models SET enabled_flag = false " + "WHERE provider_account_id = %s AND NOT (provider_model_id = ANY(%s))", + (account.provider_account_id, seen_ids), + ) + else: + cursor.execute( + "UPDATE provider_models SET enabled_flag = false " + "WHERE provider_account_id = %s", + (account.provider_account_id,), + ) + _insert_refresh_row( + cursor, + account.provider_account_id, + "refreshed", + len(unique), + None, + started_at, + _utc_now(), + ) + connection.commit() + + def record_failure(self, account: ProviderAccount, error_code: str) -> None: + started_at = _utc_now() + with self._connect() as connection: + self._ensure_schema(connection) + with connection.cursor() as cursor: + _upsert_account_row(cursor, account) + _insert_refresh_row( + cursor, + account.provider_account_id, + "failed", + 0, + error_code, + started_at, + _utc_now(), + ) + connection.commit() + + def enabled_models(self) -> list[CatalogModelRecord]: + return self._read_models(enabled_accounts_only=True) + + def all_models(self) -> list[CatalogModelRecord]: + return self._read_models(enabled_accounts_only=False) + + def has_models(self, provider_account_id: str) -> bool: + with self._connect() as connection: + self._ensure_schema(connection) + with connection.cursor() as cursor: + cursor.execute( + "SELECT EXISTS (SELECT 1 FROM provider_models " + "WHERE provider_account_id = %s AND enabled_flag = true)", + (provider_account_id,), + ) + row = cursor.fetchone() + return bool(row and row[0]) + + def _read_models(self, *, enabled_accounts_only: bool) -> list[CatalogModelRecord]: + condition = "AND a.enabled_flag = true" if enabled_accounts_only else "" + with self._connect() as connection: + self._ensure_schema(connection) + with connection.cursor() as cursor: + cursor.execute( + "SELECT m.provider_account_id, m.provider_model_id, m.model_name, " + "m.display_name, m.context_window, m.input_price_usd_per_million, " + "m.output_price_usd_per_million " + "FROM provider_models m JOIN provider_accounts a " + "ON a.provider_account_id = m.provider_account_id " + f"WHERE m.enabled_flag = true {condition} " # nosec B608 - fixed fragment + "ORDER BY m.provider_account_id, m.model_name" + ) + rows = cursor.fetchall() + records: list[CatalogModelRecord] = [] + for row in rows: + cursor.execute( + "SELECT capability_name FROM model_capabilities " + "WHERE provider_model_id = %s ORDER BY capability_name", + (row[1],), + ) + capabilities = tuple(item[0] for item in cursor.fetchall()) + cursor.execute( + "SELECT modality_name FROM model_modalities " + "WHERE provider_model_id = %s ORDER BY modality_name", + (row[1],), + ) + modalities = tuple(item[0] for item in cursor.fetchall()) + records.append( + CatalogModelRecord( + row[0], + DiscoveredModel( + model_name=row[2], + display_name=row[3], + capabilities=capabilities, + modalities=modalities, + context_window=row[4], + input_price_usd_per_million=_optional_float(row[5]), + output_price_usd_per_million=_optional_float(row[6]), + ), + ) + ) + return records + + +class _PinnedCatalogConnection(http.client.HTTPSConnection): # pragma: no cover - network adapter + """Connect to a validated address while retaining hostname TLS verification.""" + + def __init__(self, hostname: str, pinned_ip: str, port: int, timeout: float, context: ssl.SSLContext) -> None: + super().__init__(hostname, port=port, timeout=timeout, context=context) + self._pinned_ip = pinned_ip + self._catalog_hostname = hostname + + def connect(self) -> None: + raw_socket = socket.create_connection((self._pinned_ip, self.port), self.timeout) + try: + self.sock = self._context.wrap_socket(raw_socket, server_hostname=self._catalog_hostname) + except Exception: + raw_socket.close() + raise + + +class ProviderCatalogHttpClient: + """Bounded DNS-pinned HTTPS client for provider model listings.""" + + TRANSIENT_STATUS = frozenset({408, 409, 425, 429, 500, 502, 503, 504}) + + def __init__( + self, + *, + timeout_seconds: float = 20.0, + max_attempts: int = 3, + deadline_seconds: float = 60.0, + sleep: Callable[[float], None] = time.sleep, + random_uniform: Callable[[float, float], float] = random.uniform, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if timeout_seconds <= 0 or max_attempts < 1 or deadline_seconds <= 0: + raise ValueError("catalog HTTP limits must be positive") + self.timeout_seconds = timeout_seconds + self.max_attempts = max_attempts + self.deadline_seconds = deadline_seconds + self._sleep = sleep + self._random_uniform = random_uniform + self._clock = clock + self._ssl_context = ssl.create_default_context() + + def discover(self, account: ProviderAccount, credential: str) -> list[DiscoveredModel]: + """Fetch and normalize one account's model document with bounded retries.""" + if account.models_url is None: + raise CatalogHttpError("catalog_endpoint_not_configured") + started = self._clock() + for attempt in range(self.max_attempts): + if self._clock() - started >= self.deadline_seconds: + raise CatalogHttpError("catalog_deadline_exceeded", transient=True) + try: + document = self._request_json(account, credential) + models = normalize_models_document(document) + if not models: + raise CatalogHttpError("catalog_contains_no_models") + return models + except CatalogHttpError as exc: + if not exc.transient or attempt + 1 >= self.max_attempts: + raise + ceiling = min(8.0, 0.5 * (2**attempt)) + self._sleep(self._random_uniform(0.0, ceiling)) + raise CatalogHttpError("catalog_attempts_exhausted", transient=True) + + def _request_json(self, account: ProviderAccount, credential: str) -> dict[str, Any]: # pragma: no cover - network + return _secure_json_request( + method="GET", + url=account.models_url or "", + header_name=account.auth_header_name, + authorization=f"{account.auth_prefix} {credential}".strip(), + payload=None, + timeout_seconds=self.timeout_seconds, + transient_status=self.TRANSIENT_STATUS, + ) + + +class ProviderAwareModelClient(ModelClient): + """Use the existing OpenAI transport plus a narrow native Bytez adapter.""" + + def __init__( + self, + *args: Any, + bytez_request: Callable[[ModelAgent, list[dict[str, str]], str], Mapping[str, Any]] | None = None, + **kwargs: Any, + ) -> None: + super().__init__(*args, **kwargs) + self._bytez_request = bytez_request or self._request_bytez + + def chat(self, agent: ModelAgent, messages: list[dict[str, str]], temperature: float = 0.2) -> str: + """Dispatch Bytez through its native Key/input contract and delegate all peers.""" + if agent.provider_name != "bytez": + return super().chat(agent, messages, temperature=temperature) + self._local.usage = None + credential = get_credential(agent.credential_name) + if not credential: + raise ProviderCatalogUnavailable("Bytez credential is not registered") + document = self._bytez_request(agent, messages, credential) + return _normalize_bytez_output(document) + + def stream_chat(self, agent: ModelAgent, messages: list[dict[str, str]], temperature: float = 0.2): + """Frame a completed native Bytez answer when that API offers no token SSE contract.""" + if agent.provider_name != "bytez": + yield from super().stream_chat(agent, messages, temperature=temperature) + return + answer = self.chat(agent, messages, temperature=temperature) + for start in range(0, len(answer), 24): + yield answer[start : start + 24] + + def proxy_send(self, agent: ModelAgent, endpoint: str, payload: dict[str, Any]) -> dict[str, Any]: + """Fail closed for unsupported Bytez passthrough instead of fabricating OpenAI shapes.""" + if agent.provider_name == "bytez": + raise ProviderCatalogUnavailable( + f"Bytez native transport does not support passthrough endpoint {endpoint}" + ) + return super().proxy_send(agent, endpoint, payload) + + def _request_bytez( + self, + agent: ModelAgent, + messages: list[dict[str, str]], + credential: str, + ) -> Mapping[str, Any]: # pragma: no cover - real Bytez network boundary + model_path = quote(agent.model, safe="") + return _secure_json_request( + method="POST", + url=f"{agent.base_url.rstrip('/')}/models/v2/{model_path}", + header_name="Authorization", + authorization=f"Key {credential}", + payload={"input": messages}, + timeout_seconds=float(self.timeout), + transient_status=ProviderCatalogHttpClient.TRANSIENT_STATUS, + ) + + +class ProviderCatalogService: + """Coordinate isolated provider refreshes and build the runtime agent pool.""" + + def __init__( + self, + *, + store: ProviderCatalogStore, + accounts: Sequence[ProviderAccount] = DEFAULT_PROVIDER_ACCOUNTS, + discover: Callable[[ProviderAccount, str], Sequence[DiscoveredModel]] | None = None, + ) -> None: + self.store = store + self.accounts = tuple(accounts) + self._account_by_id = {account.provider_account_id: account for account in self.accounts} + self._discover = discover or ProviderCatalogHttpClient().discover + self.last_refresh_summary: dict[str, Any] = { + "provider_accounts": {}, + "candidate_model_count": 0, + "measurement_status": "provider_catalog_snapshot", + } + + def refresh_all(self) -> dict[str, Any]: + """Refresh each account independently and preserve stale usable catalogs.""" + provider_rows: dict[str, dict[str, Any]] = {} + for account in self.accounts: + self.store.upsert_account(account) + if not account.enabled: + provider_rows[account.provider_account_id] = { + "status": "disabled", + "model_count": 0, + "error_code": None, + } + continue + credential = get_credential(account.credential_name) + if not credential: + provider_rows[account.provider_account_id] = self._failed_refresh( + account, "credential_not_registered" + ) + continue + try: + models = list(self._discover(account, credential)) + if not models: + raise CatalogHttpError("catalog_contains_no_models") + self.store.replace_catalog(account, models) + provider_rows[account.provider_account_id] = { + "status": "refreshed", + "model_count": len(models), + "error_code": None, + } + except CatalogHttpError as exc: + provider_rows[account.provider_account_id] = self._failed_refresh(account, exc.code) + except Exception: + provider_rows[account.provider_account_id] = self._failed_refresh( + account, "catalog_adapter_failure" + ) + candidates = self.store.enabled_models() + self.last_refresh_summary = { + "provider_accounts": provider_rows, + "candidate_model_count": len(candidates), + "measurement_status": "provider_catalog_snapshot", + } + if not candidates: + raise ProviderCatalogUnavailable("no usable provider model exists after catalog refresh") + return self.last_refresh_summary + + def _failed_refresh(self, account: ProviderAccount, code: str) -> dict[str, Any]: + """Record failure and classify whether last-known-good service remains available.""" + self.store.record_failure(account, code) + stale_available = self.store.has_models(account.provider_account_id) + return { + "status": "stale_available" if stale_available else "failed", + "model_count": 0, + "error_code": code, + } + + def candidate_agents(self) -> list[ModelAgent]: + """Convert enabled catalog rows into role-tagged, failover-capable agents.""" + agents: list[ModelAgent] = [] + for record in self.store.enabled_models(): + account = self._account_by_id[record.provider_account_id] + model = record.model + agents.append( + ModelAgent( + id=_agent_id(account.provider_account_id, model.model_name), + model=model.model_name, + base_url=account.base_url, + credential_key=account.credential_name, + tags=_agent_tags(model), + priority=account.priority_rank + _model_priority(model), + provider_name=account.provider_name, + ) + ) + return agents + + +def bootstrap_provider_credentials( + environment: Mapping[str, str], + *, + require_all: bool, + accounts: Sequence[ProviderAccount] = DEFAULT_PROVIDER_ACCOUNTS, +) -> dict[str, list[str]]: + """Transport the fixed provider-secret inventory into the credential registry. + + Validation happens before mutation when ``require_all`` is true, preventing a + partially updated production credential set. The returned summary contains + names only and is safe for CI logs. + """ + values = { + account.credential_name: str(environment.get(account.credential_name, "")).strip() + for account in accounts + } + missing = [name for name, value in values.items() if not value] + if require_all and missing: + raise ProviderCatalogUnavailable("provider credential inventory is incomplete") + registered: list[str] = [] + for account in accounts: + value = values[account.credential_name] + if value: + register_credential(account.credential_name, value) + registered.append(account.credential_name) + return {"registered_credentials": registered, "missing_credentials": missing} + + +def normalize_models_document(document: Mapping[str, Any]) -> list[DiscoveredModel]: + """Normalize common OpenAI/OpenRouter/Bytez listing shapes into model rows.""" + raw_rows: Any = document.get("data") + if not isinstance(raw_rows, list): + raw_rows = document.get("models") + if isinstance(raw_rows, Mapping): + raw_rows = list(raw_rows.values()) + if not isinstance(raw_rows, list): + return [] + models: dict[str, DiscoveredModel] = {} + for raw in raw_rows: + if isinstance(raw, str): + raw = {"id": raw} + if not isinstance(raw, Mapping): + continue + name = str(raw.get("id") or raw.get("model") or raw.get("name") or "").strip() + if not name or len(name) > 512: + continue + display_name = str(raw.get("name") or raw.get("display_name") or name).strip()[:512] or name + architecture = raw.get("architecture") if isinstance(raw.get("architecture"), Mapping) else {} + input_modalities = _string_values( + architecture.get("input_modalities") or raw.get("input_modalities") or raw.get("modalities") + ) + output_modalities = _string_values( + architecture.get("output_modalities") or raw.get("output_modalities") + ) + modalities = tuple(sorted(set(input_modalities + output_modalities) or {"text"})) + capabilities = _infer_capabilities(name, raw, modalities) + context_window = _optional_positive_int( + raw.get("context_length") or raw.get("context_window") or raw.get("max_context_length") + ) + pricing = raw.get("pricing") if isinstance(raw.get("pricing"), Mapping) else {} + input_price = _per_token_price_to_million( + pricing.get("prompt") or raw.get("input_price_per_token") + ) + output_price = _per_token_price_to_million( + pricing.get("completion") or raw.get("output_price_per_token") + ) + models[name] = DiscoveredModel( + model_name=name, + display_name=display_name, + capabilities=capabilities, + modalities=modalities, + context_window=context_window, + input_price_usd_per_million=input_price, + output_price_usd_per_million=output_price, + ) + return [models[name] for name in sorted(models)] + + +def build_catalog_orchestrator( + store: ProviderCatalogStore, + *, + accounts: Sequence[ProviderAccount] = DEFAULT_PROVIDER_ACCOUNTS, + client: ModelClient | None = None, + **orchestrator_options: Any, +) -> TaskOrchestrator: + """Build a normal :class:`TaskOrchestrator` from the durable candidate pool.""" + service = ProviderCatalogService(store=store, accounts=accounts) + agents = service.candidate_agents() + if not agents: + raise ProviderCatalogUnavailable("provider catalog contains no enabled agents") + return TaskOrchestrator( + agents, + client=client or ProviderAwareModelClient(), + **orchestrator_options, + ) + + +def _normalize_bytez_output(document: Mapping[str, Any]) -> str: + """Extract text from the bounded native Bytez response contract.""" + output = document.get("output") + if isinstance(output, str) and output: + return output + if isinstance(output, Mapping): + content = output.get("content") or output.get("text") + if isinstance(content, str) and content: + return content + raise ProviderCatalogUnavailable("Bytez response shape is unsupported") + + +def _secure_json_request( # pragma: no cover - real credentialed network boundary + *, + method: str, + url: str, + header_name: str, + authorization: str, + payload: Mapping[str, Any] | None, + timeout_seconds: float, + transient_status: Sequence[int], +) -> dict[str, Any]: + parsed = urlparse(url) + if parsed.scheme != "https" or not parsed.hostname: + raise CatalogHttpError("catalog_url_must_use_https") + if parsed.username is not None or parsed.password is not None: + raise CatalogHttpError("catalog_url_must_not_contain_userinfo") + port = parsed.port or 443 + addresses = _validated_global_addresses(parsed.hostname, port) + target = parsed.path or "/" + if parsed.query: + target = f"{target}?{parsed.query}" + body = None if payload is None else json.dumps(payload).encode("utf-8") + headers = { + header_name: authorization, + "Accept": "application/json", + "Connection": "close", + "User-Agent": "contextual-orchestrator-provider-catalog/1", + } + if body is not None: + headers["Content-Type"] = "application/json" + last_network_error: BaseException | None = None + for address in addresses: + connection = _PinnedCatalogConnection( + parsed.hostname, + address, + port, + timeout_seconds, + ssl.create_default_context(), + ) + try: + connection.request(method, target, body=body, headers=headers) + response = connection.getresponse() + status = response.status + if status >= 300: + response.close() + connection.close() + if status in {401, 403}: + raise CatalogHttpError("catalog_authentication_failed") + raise CatalogHttpError( + f"catalog_http_{status}", transient=status in transient_status + ) + content_type = (response.getheader("Content-Type") or "").lower() + if "json" not in content_type: + response.close() + connection.close() + raise CatalogHttpError("catalog_content_type_invalid") + raw_payload = response.read(CATALOG_RESPONSE_MAX_BYTES + 1) + response.close() + connection.close() + if len(raw_payload) > CATALOG_RESPONSE_MAX_BYTES: + raise CatalogHttpError("catalog_response_too_large") + try: + document = json.loads(raw_payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError, RecursionError): + raise CatalogHttpError("catalog_json_invalid") from None + if not isinstance(document, dict): + raise CatalogHttpError("catalog_json_must_be_object") + return document + except CatalogHttpError: + raise + except (OSError, http.client.HTTPException, TimeoutError) as exc: + connection.close() + last_network_error = exc + raise CatalogHttpError("catalog_network_failure", transient=True) from last_network_error + + +def _upsert_account_row(cursor: Any, account: ProviderAccount) -> None: # pragma: no cover - SQL adapter + """Execute the parameter-bound provider-account upsert.""" + cursor.execute( + "INSERT INTO provider_accounts (" + "provider_account_id, provider_name, credential_name, base_url, models_path, " + "transport_name, auth_header_name, auth_prefix, enabled_flag, priority_rank, updated_at) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, now()) " + "ON CONFLICT (provider_account_id) DO UPDATE SET " + "provider_name = EXCLUDED.provider_name, credential_name = EXCLUDED.credential_name, " + "base_url = EXCLUDED.base_url, models_path = EXCLUDED.models_path, " + "transport_name = EXCLUDED.transport_name, auth_header_name = EXCLUDED.auth_header_name, " + "auth_prefix = EXCLUDED.auth_prefix, enabled_flag = EXCLUDED.enabled_flag, " + "priority_rank = EXCLUDED.priority_rank, updated_at = now()", + ( + account.provider_account_id, + account.provider_name, + account.credential_name, + account.base_url, + account.models_path, + account.transport_name, + account.auth_header_name, + account.auth_prefix, + account.enabled, + account.priority_rank, + ), + ) + + +def _insert_refresh_row( # pragma: no cover - SQL adapter + cursor: Any, + account_id: str, + status: str, + count: int, + error_code: str | None, + started_at: datetime, + finished_at: datetime, +) -> None: + """Insert one immutable provider refresh evidence row.""" + cursor.execute( + "INSERT INTO catalog_refresh_runs (" + "catalog_refresh_id, provider_account_id, refresh_status, observed_model_count, " + "error_code, started_at, finished_at) VALUES (%s, %s, %s, %s, %s, %s, %s)", + ( + _refresh_id(account_id, finished_at.isoformat()), + account_id, + status, + count, + error_code, + started_at, + finished_at, + ), + ) + + +def _validated_global_addresses(hostname: str, port: int) -> tuple[str, ...]: # pragma: no cover - DNS boundary + """Resolve and accept only globally routable addresses for credentialed egress.""" + addresses: list[str] = [] + try: + candidates = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + except socket.gaierror: + raise CatalogHttpError("catalog_dns_failure", transient=True) from None + for candidate in candidates: + address = ipaddress.ip_address(candidate[4][0]) + if ( + not address.is_global + or address.is_private + or address.is_loopback + or address.is_link_local + or address.is_multicast + or address.is_reserved + ): + raise CatalogHttpError("catalog_destination_not_public") + value = str(address) + if value not in addresses: + addresses.append(value) + if not addresses: + raise CatalogHttpError("catalog_dns_empty", transient=True) + return tuple(addresses) + + +def _infer_capabilities( + model_name: str, + raw: Mapping[str, Any], + modalities: Sequence[str], +) -> tuple[str, ...]: + """Infer conservative routing tags from provider metadata and model naming.""" + lowered = model_name.lower() + capabilities = {value.lower() for value in _string_values(raw.get("capabilities"))} + if any(token in lowered for token in ("embed", "embedding")): + capabilities.add("embeddings") + elif "rerank" in lowered: + capabilities.add("reranking") + elif "moderation" in lowered: + capabilities.add("moderation") + else: + capabilities.add("chat") + if any(token in lowered for token in ("reason", "o1", "o3", "r1", "thinking")): + capabilities.add("reasoning") + if any(token in lowered for token in ("code", "coder", "codestral", "devstral")): + capabilities.add("coding") + if "image" in modalities or "vision" in lowered or "vl" in lowered: + capabilities.add("vision") + if "audio" in modalities or any(token in lowered for token in ("audio", "whisper", "speech")): + capabilities.add("audio") + if "guard" in lowered: + capabilities.add("moderation") + return tuple(sorted(capabilities)) + + +def _agent_tags(model: DiscoveredModel) -> tuple[str, ...]: + """Map provider capabilities into the orchestrator's role/domain tag vocabulary.""" + tags: set[str] = set(model.capabilities) + if "chat" in tags: + tags.update(("writing", "summarization", "classification")) + if "reasoning" in tags: + tags.update(("planning", "research", "verification")) + if "coding" in tags: + tags.update(("implementation", "debugging")) + if "vision" in tags: + tags.update(("image", "multimodal")) + if "audio" in tags: + tags.update(("speech", "multimodal")) + return tuple(sorted(tags)) + + +def _model_priority(model: DiscoveredModel) -> int: + """Use context and known price only as small ties after role/capability scoring.""" + score = min(3, (model.context_window or 0) // 100_000) + known_prices = [ + value + for value in ( + model.input_price_usd_per_million, + model.output_price_usd_per_million, + ) + if value is not None + ] + if known_prices: + average = sum(known_prices) / len(known_prices) + score += max(0, 2 - min(2, int(average))) + return score + + +def _agent_id(provider_account_id: str, model_name: str) -> str: + """Create a bounded two-or-more-word snake-case agent identifier.""" + slug = re.sub(r"[^a-z0-9]+", "_", model_name.lower()).strip("_") or "model_worker" + digest = hashlib.sha256(model_name.encode("utf-8")).hexdigest()[:8] + return f"{provider_account_id}_{slug}_{digest}"[:120].rstrip("_") + + +def _provider_model_id(provider_account_id: str, model_name: str) -> str: # pragma: no cover - SQL adapter + """Return a stable non-secret identifier for one account/model pair.""" + material = f"{provider_account_id}\0{model_name}".encode("utf-8") + return f"provider_model_{hashlib.sha256(material).hexdigest()}" + + +def _refresh_id(account_id: str, timestamp: str) -> str: + """Return an immutable refresh identifier without exposing credentials.""" + material = f"{account_id}\0{timestamp}".encode("utf-8") + return f"catalog_refresh_{hashlib.sha256(material).hexdigest()}" + + +def _string_values(value: Any) -> list[str]: + """Return bounded, non-empty strings from scalar or sequence metadata.""" + if isinstance(value, str): + values: Iterable[Any] = (value,) + elif isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + values = value + else: + return [] + result: list[str] = [] + for item in values: + if isinstance(item, str): + normalized = item.strip().lower() + if normalized and len(normalized) <= 128: + result.append(normalized) + return result + + +def _optional_positive_int(value: Any) -> int | None: + """Return a positive integer metadata value, rejecting booleans and overflow.""" + if isinstance(value, bool) or value is None: + return None + try: + parsed = int(value) + except (TypeError, ValueError, OverflowError): + return None + return parsed if 0 < parsed <= 10_000_000_000 else None + + +def _per_token_price_to_million(value: Any) -> float | None: + """Convert a finite non-negative per-token USD price to per-million units.""" + if isinstance(value, bool) or value is None: + return None + try: + parsed = float(value) + except (TypeError, ValueError, OverflowError): + return None + if not math.isfinite(parsed) or parsed < 0: + return None + return parsed * 1_000_000 + + +def _optional_float(value: Any) -> float | None: # pragma: no cover - SQL adapter + """Convert a finite database numeric value to float, preserving null.""" + if value is None: + return None + parsed = float(value) + return parsed if math.isfinite(parsed) else None + + +def _utc_now() -> datetime: + """Return the current timezone-aware UTC timestamp.""" + return datetime.now(timezone.utc) + + +def _utc_now_text() -> str: + """Return the current UTC timestamp as an ISO-8601 string.""" + return _utc_now().isoformat() + + +def _safe_cli_summary( # pragma: no cover - CLI integration + credential_summary: Mapping[str, Any], catalog_summary: Mapping[str, Any] +) -> dict[str, Any]: + """Build a log-safe bootstrap summary containing no credential values.""" + return { + "registered_credentials": list(credential_summary.get("registered_credentials", [])), + "missing_credentials": list(credential_summary.get("missing_credentials", [])), + "candidate_model_count": int(catalog_summary.get("candidate_model_count", 0)), + "provider_accounts": dict(catalog_summary.get("provider_accounts", {})), + "measurement_status": "provider_catalog_bootstrap", + } + + +def _write_agents_file(path: str, agents: Sequence[ModelAgent]) -> None: # pragma: no cover - CLI integration + """Atomically write a secret-free agent configuration JSON document.""" + target = os.path.abspath(path) + os.makedirs(os.path.dirname(target) or ".", exist_ok=True) + temporary = f"{target}.tmp-{os.getpid()}" + try: + with open(temporary, "w", encoding="utf-8") as handle: + json.dump( + {"agents": [agent.to_config() for agent in agents]}, + handle, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + handle.write("\n") + os.replace(temporary, target) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def main(argv: Sequence[str] | None = None) -> int: # pragma: no cover - CLI integration + """Bootstrap credentials, refresh the durable catalog, and optionally export agents.""" + parser = argparse.ArgumentParser(description="Bootstrap and refresh the durable provider catalog.") + parser.add_argument("command", choices=("bootstrap-and-sync", "sync", "export-agents")) + parser.add_argument( + "--catalog-dsn", + default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN") + or os.environ.get("CONTEXTUAL_ORCHESTRATOR_KV_DSN", ""), + help="PostgreSQL DSN used for provider metadata (bootstrap transport only).", + ) + parser.add_argument("--require-all", action="store_true") + parser.add_argument("--agents-output", default="") + args = parser.parse_args(list(argv) if argv is not None else None) + + store = PostgresProviderCatalogStore(args.catalog_dsn) + credential_summary: dict[str, list[str]] = { + "registered_credentials": [], + "missing_credentials": [], + } + if args.command == "bootstrap-and-sync": + credential_summary = bootstrap_provider_credentials(os.environ, require_all=args.require_all) + service = ProviderCatalogService(store=store) + if args.command in {"bootstrap-and-sync", "sync"}: + catalog_summary = service.refresh_all() + else: + catalog_summary = { + "candidate_model_count": len(store.enabled_models()), + "provider_accounts": {}, + } + agents = service.candidate_agents() + if not agents: + raise ProviderCatalogUnavailable("provider catalog contains no enabled agents") + if args.agents_output: + _write_agents_file(args.agents_output, agents) + print(json.dumps(_safe_cli_summary(credential_summary, catalog_summary), sort_keys=True)) + return 0 + + +if __name__ == "__main__": # pragma: no cover - CLI integration + try: + raise SystemExit(main()) + except ProviderCatalogUnavailable as exc: + print( + json.dumps({"error": "provider_catalog_unavailable", "message": str(exc)}), + file=sys.stderr, + ) + raise SystemExit(2) from None From af5381ef5f7ffe161494daec91865dd859feefa7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:44:29 +0900 Subject: [PATCH 03/15] test: close provider catalog branch coverage --- tests/test_provider_catalog_coverage.py | 180 ++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 tests/test_provider_catalog_coverage.py diff --git a/tests/test_provider_catalog_coverage.py b/tests/test_provider_catalog_coverage.py new file mode 100644 index 00000000..5d351108 --- /dev/null +++ b/tests/test_provider_catalog_coverage.py @@ -0,0 +1,180 @@ +"""Focused branch coverage for provider-catalog boundary helpers.""" + +from __future__ import annotations + +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import contextual_orchestrator.provider_catalog as catalog # noqa: E402 +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + set_backend, +) +from contextual_orchestrator.orchestrator import ModelAgent # noqa: E402 + + +@pytest.fixture(autouse=True) +def _credential_backend(): + """Use one isolated credential registry for every focused branch test.""" + set_backend(InMemoryCredentialBackend()) + try: + yield + finally: + set_backend(None) + + +def test_provider_account_can_explicitly_disable_catalog_discovery() -> None: + """An account without a listing endpoint advertises no models URL.""" + account = catalog.ProviderAccount( + "custom_provider", + "custom_provider", + "CUSTOM_PROVIDER_KEY", + "https://models.example", + models_path=None, + ) + assert account.models_url is None + client = catalog.ProviderCatalogHttpClient() + with pytest.raises(catalog.CatalogHttpError, match="catalog_endpoint_not_configured"): + client.discover(account, "credential") + + +def test_http_limit_validation_and_deadline_failure() -> None: + """Invalid limits and an exhausted wall-clock deadline fail before network access.""" + for options in ( + {"timeout_seconds": 0}, + {"max_attempts": 0}, + {"deadline_seconds": 0}, + ): + with pytest.raises(ValueError, match="limits must be positive"): + catalog.ProviderCatalogHttpClient(**options) + + ticks = iter((10.0, 11.0)) + client = catalog.ProviderCatalogHttpClient(deadline_seconds=0.5, clock=lambda: next(ticks)) + with pytest.raises(catalog.CatalogHttpError, match="catalog_deadline_exceeded"): + client.discover(catalog.DEFAULT_PROVIDER_ACCOUNTS[0], "credential") + + +def test_http_attempts_exhausted_guard_is_stable(monkeypatch: pytest.MonkeyPatch) -> None: + """The defensive post-loop guard retains a stable secret-free error code.""" + client = catalog.ProviderCatalogHttpClient() + monkeypatch.setattr(catalog, "range", lambda _count: [], raising=False) + with pytest.raises(catalog.CatalogHttpError, match="catalog_attempts_exhausted"): + client.discover(catalog.DEFAULT_PROVIDER_ACCOUNTS[0], "credential") + + +def test_model_normalization_covers_specialized_capabilities_and_bad_values() -> None: + """Reranking, moderation, audio, guard, and malformed metadata stay deterministic.""" + models = catalog.normalize_models_document( + { + "data": [ + {"id": "rank/rerank-large"}, + {"id": "safe/moderation-latest"}, + {"id": "voice/whisper-audio", "modalities": "audio"}, + {"id": "secure/guard-model"}, + { + "id": "invalid-metadata", + "context_length": object(), + "pricing": {"prompt": object(), "completion": True}, + "capabilities": ["", 7, "x" * 129, "CUSTOM"], + }, + {"id": "x" * 513}, + ] + } + ) + by_name = {model.model_name: model for model in models} + assert by_name["rank/rerank-large"].capabilities == ("reranking",) + assert by_name["safe/moderation-latest"].capabilities == ("moderation",) + assert by_name["voice/whisper-audio"].capabilities == ("audio", "chat") + assert by_name["secure/guard-model"].capabilities == ("chat", "moderation") + invalid = by_name["invalid-metadata"] + assert invalid.capabilities == ("chat", "custom") + assert invalid.context_window is None + assert invalid.input_price_usd_per_million is None + assert invalid.output_price_usd_per_million is None + assert len(by_name) == 5 + + +def test_candidate_tags_cover_multimodal_and_empty_slug_fallback() -> None: + """Multimodal role tags and hostile model identifiers produce valid agent records.""" + account = catalog.DEFAULT_PROVIDER_ACCOUNTS[4] + store = catalog.InMemoryProviderCatalogStore() + store.replace_catalog( + account, + [ + catalog.DiscoveredModel( + model_name="!!!", + display_name="Punctuation", + capabilities=("chat", "coding", "vision", "audio"), + modalities=("audio", "image", "text"), + context_window=300_000, + input_price_usd_per_million=0.0, + output_price_usd_per_million=0.0, + ) + ], + ) + agent = catalog.ProviderCatalogService(store=store, accounts=(account,)).candidate_agents()[0] + assert "model_worker" in agent.id + assert {"implementation", "debugging", "image", "speech", "multimodal"}.issubset(agent.tags) + assert agent.priority == account.priority_rank + 5 + + +def test_empty_catalog_factory_fails_closed() -> None: + """Runtime construction never falls back to an implicit mock worker.""" + with pytest.raises(catalog.ProviderCatalogUnavailable, match="no enabled agents"): + catalog.build_catalog_orchestrator(catalog.InMemoryProviderCatalogStore()) + + +def test_bytez_string_output_streaming_and_passthrough_guard() -> None: + """Native Bytez text can be framed, while unsupported passthrough fails closed.""" + account = catalog.DEFAULT_PROVIDER_ACCOUNTS[2] + catalog.bootstrap_provider_credentials( + {account.credential_name: "bytez-secret"}, + require_all=False, + accounts=(account,), + ) + agent = ModelAgent( + "bytez_worker", + "owner/model", + account.base_url, + credential_key=account.credential_name, + provider_name="bytez", + ) + client = catalog.ProviderAwareModelClient( + bytez_request=lambda _agent, _messages, _credential: {"output": "native-answer"} + ) + assert "".join(client.stream_chat(agent, [{"role": "user", "content": "hello"}])) == "native-answer" + with pytest.raises(catalog.ProviderCatalogUnavailable, match="does not support passthrough"): + client.proxy_send(agent, "/responses", {}) + + +def test_non_bytez_stream_and_proxy_keep_existing_mock_behavior() -> None: + """Provider-aware delegation preserves the existing mock transport surfaces.""" + client = catalog.ProviderAwareModelClient() + agent = ModelAgent("general_agent", "mock-generalist", "mock://local") + chunks = list(client.stream_chat(agent, [{"role": "user", "content": "hello"}])) + assert "".join(chunks) + raw = client.proxy_send(agent, "/responses", {"input": "hello"}) + assert isinstance(raw, dict) + + +def test_scalar_capability_and_extreme_context_helpers() -> None: + """Scalar provider metadata and oversized integer values are bounded.""" + model = catalog.normalize_models_document( + { + "data": [ + { + "id": "custom-model", + "capabilities": "SPECIAL", + "context_length": "10000000001", + "pricing": {"prompt": "not-a-number"}, + } + ] + } + )[0] + assert model.capabilities == ("chat", "special") + assert model.context_window is None + assert model.input_price_usd_per_million is None From 04ffda260d6408241980bb45c0efe171301e8585 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:44:54 +0900 Subject: [PATCH 04/15] ci: add trusted provider catalog bootstrap --- .github/workflows/provider-catalog-sync.yml | 146 ++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 .github/workflows/provider-catalog-sync.yml diff --git a/.github/workflows/provider-catalog-sync.yml b/.github/workflows/provider-catalog-sync.yml new file mode 100644 index 00000000..d4357b4c --- /dev/null +++ b/.github/workflows/provider-catalog-sync.yml @@ -0,0 +1,146 @@ +name: Provider Catalog Sync + +on: + pull_request: + workflow_dispatch: + schedule: + - cron: "17 */6 * * *" + +permissions: + contents: read + +concurrency: + group: provider-catalog-${{ github.ref }} + cancel-in-progress: false + +jobs: + contract: + name: Offline provider-catalog contracts + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout exact revision + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install hash-locked test dependencies + run: | + python -m pip install --require-hashes -r requirements-opencode-review-ci.txt + python -m pip install --require-hashes -r fuzz/requirements-property.txt + + - name: Run provider-catalog contracts + run: | + python -m pytest tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py -q + python -m compileall -q contextual_orchestrator + + synchronize: + name: Seed credentials and refresh durable catalog + if: >- + github.event_name != 'pull_request' && + github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: production + env: + CONTEXTUAL_ORCHESTRATOR_KV_BACKEND: postgres + CONTEXTUAL_ORCHESTRATOR_KV_DSN: ${{ secrets.CONTEXTUAL_ORCHESTRATOR_KV_DSN }} + CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN: ${{ secrets.CONTEXTUAL_ORCHESTRATOR_KV_DSN }} + CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE: ${{ secrets.CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE }} + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} + BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + steps: + - name: Checkout protected default-branch revision + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install hash-locked runtime and database dependencies + run: python -m pip install --require-hashes -r requirements.lock + + - name: Validate trusted bootstrap inventory + shell: bash + run: | + set +x + required=( + CONTEXTUAL_ORCHESTRATOR_KV_DSN + CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE + NVIDIA_NIM_API_KEY + NVIDIA_NIM_API_KEY_SUB + BYTEZ_API_KEY + OPENROUTER_API_KEY + OPENAI_API_KEY + ) + for name in "${required[@]}"; do + value="${!name:-}" + if [[ -z "$value" ]]; then + echo "::error title=Provider catalog bootstrap blocked::Required secret $name is not configured" + exit 2 + fi + echo "::add-mask::$value" + done + + - name: Seed encrypted credential registry and refresh model catalog + shell: bash + run: | + set +x + python -m contextual_orchestrator.provider_catalog \ + bootstrap-and-sync \ + --require-all \ + --agents-output "$RUNNER_TEMP/provider-agents.json" \ + > "$RUNNER_TEMP/provider-catalog-summary.json" + + - name: Verify secret-free generated agent pool + shell: bash + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + agents_path = Path(os.environ["RUNNER_TEMP"]) / "provider-agents.json" + summary_path = Path(os.environ["RUNNER_TEMP"]) / "provider-catalog-summary.json" + agents = json.loads(agents_path.read_text(encoding="utf-8"))["agents"] + summary = json.loads(summary_path.read_text(encoding="utf-8")) + if not agents: + raise SystemExit("provider catalog produced no candidate agents") + forbidden = { + os.environ[name] + for name in ( + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "BYTEZ_API_KEY", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", + ) + } + serialized = json.dumps({"agents": agents, "summary": summary}) + if any(secret and secret in serialized for secret in forbidden): + raise SystemExit("generated provider evidence contains a secret value") + print(json.dumps({ + "candidate_agent_count": len(agents), + "candidate_model_count": summary["candidate_model_count"], + "measurement_status": summary["measurement_status"], + }, sort_keys=True)) + PY + + - name: Confirm runtime secret-source boundary + shell: bash + run: | + unset NVIDIA_NIM_API_KEY NVIDIA_NIM_API_KEY_SUB BYTEZ_API_KEY OPENROUTER_API_KEY OPENAI_API_KEY + echo "Provider credentials are persisted in the encrypted KV registry; runtime resolves names only." From 45a676827e1a907d64e8aac0790d1bbd33ed914d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:45:33 +0900 Subject: [PATCH 05/15] feat: load runtime agents from durable provider catalog --- contextual_orchestrator/__main__.py | 44 +++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 5f68c3b7..7f11f467 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -9,6 +9,12 @@ from .credentials import register_credential from .orchestrator import ModelClient, TaskOrchestrator, load_agents +from .provider_catalog import ( + PostgresProviderCatalogStore, + ProviderAwareModelClient, + ProviderCatalogService, + ProviderCatalogUnavailable, +) from .server import SecurityConfig, serve @@ -55,6 +61,22 @@ def _register_credential_command(argv: list[str]) -> None: print(json.dumps({"registered": args.name, "backend": "kv"}, ensure_ascii=False)) +def _runtime_agents(parser: argparse.ArgumentParser, args: argparse.Namespace): + """Load either the durable discovered pool or the explicit seed-file pool.""" + if not args.provider_catalog_dsn: + return load_agents(args.agents) + try: + store = PostgresProviderCatalogStore(args.provider_catalog_dsn) + agents = ProviderCatalogService(store=store).candidate_agents() + except ProviderCatalogUnavailable as exc: + parser.error(str(exc)) + if not agents: + parser.error( + "provider catalog contains no enabled candidates; run the trusted provider-catalog sync first" + ) + return agents + + def main() -> None: """Parse CLI options and run bootstrap, prompt completion, or the HTTP server.""" if len(sys.argv) > 1 and sys.argv[1] == "register-credential": @@ -64,6 +86,14 @@ def main() -> None: parser = argparse.ArgumentParser(description="Route or conduct chat requests across model agents.") parser.add_argument("prompt", nargs="?", help="User prompt for CLI mode.") parser.add_argument("--agents", default="examples/agents.mock.json", help="Agent config JSON.") + parser.add_argument( + "--provider-catalog-dsn", + default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN") or None, + help=( + "Optional PostgreSQL provider-catalog DSN. When set, discovered enabled models " + "replace the seed agent file and provider credentials resolve from the KV registry." + ), + ) parser.add_argument("--state-db", default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_STATE_DB", "") or None, help="Optional sqlite path to persist runs/audit/analytics across restarts (default: in-memory).") parser.add_argument("--mode", choices=["auto", "route", "conduct"], default="auto") @@ -94,9 +124,19 @@ def main() -> None: help="Measure orchestration vs a single-worker baseline on these prompts and print the report.") args = parser.parse_args() - client = ModelClient(ca_bundle=args.provider_ca_bundle, verify_tls=not args.insecure_skip_tls_verify) + agents = _runtime_agents(parser, args) + if args.provider_catalog_dsn: + client = ProviderAwareModelClient( + ca_bundle=args.provider_ca_bundle, + verify_tls=not args.insecure_skip_tls_verify, + ) + else: + client = ModelClient( + ca_bundle=args.provider_ca_bundle, + verify_tls=not args.insecure_skip_tls_verify, + ) orchestrator = TaskOrchestrator( - load_agents(args.agents), + agents, client=client, state_db=args.state_db, agents_db=args.agents_db, From f7456b204211b5467a090a9f1740d576c5327aae Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:46:06 +0900 Subject: [PATCH 06/15] test: cover catalog-backed CLI startup --- tests/test_provider_catalog_cli.py | 155 +++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 tests/test_provider_catalog_cli.py diff --git a/tests/test_provider_catalog_cli.py b/tests/test_provider_catalog_cli.py new file mode 100644 index 00000000..dadbdf57 --- /dev/null +++ b/tests/test_provider_catalog_cli.py @@ -0,0 +1,155 @@ +"""CLI wiring for catalog-backed provider discovery and runtime startup.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import contextual_orchestrator.__main__ as cli # noqa: E402 +from contextual_orchestrator import ModelAgent # noqa: E402 +from contextual_orchestrator.provider_catalog import ProviderCatalogUnavailable # noqa: E402 + + +class _Parser: + """Small parser double that records fail-closed usage errors.""" + + def error(self, message: str) -> None: + """Raise a deterministic exception carrying the parser error text.""" + raise ValueError(message) + + +class _RuntimeOrchestrator: + """CLI-facing orchestrator double for catalog startup tests.""" + + instances: list["_RuntimeOrchestrator"] = [] + + def __init__(self, agents, **kwargs) -> None: + self.agents = agents + self.kwargs = kwargs + self.complete_calls: list[tuple[list[dict[str, str]], str]] = [] + type(self).instances.append(self) + + def complete(self, messages, mode="auto"): + """Record one completion and return a deterministic response.""" + self.complete_calls.append((messages, mode)) + return {"answer": "catalog-answer", "mode": mode} + + def compare_to_baseline(self, prompts, mode="auto"): + """Return a deterministic evaluation response for interface completeness.""" + return {"prompts": prompts, "mode": mode} + + +def _catalog_agent() -> ModelAgent: + """Return one valid discovered agent fixture.""" + return ModelAgent( + "openai_catalog_agent", + "catalog-model", + "https://api.openai.com/v1", + credential_key="OPENAI_API_KEY", + provider_name="openai", + ) + + +def test_runtime_agents_uses_seed_loader_without_catalog(monkeypatch: pytest.MonkeyPatch) -> None: + """The explicit seed-file path remains unchanged when no durable DSN is selected.""" + expected = [_catalog_agent()] + monkeypatch.setattr(cli, "load_agents", lambda path: expected if path == "agents.json" else []) + args = argparse.Namespace(provider_catalog_dsn=None, agents="agents.json") + assert cli._runtime_agents(_Parser(), args) is expected + + +def test_runtime_agents_loads_catalog_candidates(monkeypatch: pytest.MonkeyPatch) -> None: + """A configured durable DSN replaces the seed file with enabled catalog models.""" + expected = [_catalog_agent()] + stores: list[str] = [] + + class _Store: + def __init__(self, dsn: str) -> None: + stores.append(dsn) + + class _Service: + def __init__(self, *, store) -> None: + self.store = store + + def candidate_agents(self): + return expected + + monkeypatch.setattr(cli, "PostgresProviderCatalogStore", _Store) + monkeypatch.setattr(cli, "ProviderCatalogService", _Service) + args = argparse.Namespace(provider_catalog_dsn="postgresql://catalog", agents="ignored.json") + + assert cli._runtime_agents(_Parser(), args) is expected + assert stores == ["postgresql://catalog"] + + +def test_runtime_agents_reports_catalog_initialization_failure(monkeypatch: pytest.MonkeyPatch) -> None: + """Durable catalog errors reach argparse without a silent seed or memory fallback.""" + monkeypatch.setattr( + cli, + "PostgresProviderCatalogStore", + lambda _dsn: (_ for _ in ()).throw(ProviderCatalogUnavailable("catalog unavailable")), + ) + args = argparse.Namespace(provider_catalog_dsn="postgresql://catalog", agents="ignored.json") + with pytest.raises(ValueError, match="catalog unavailable"): + cli._runtime_agents(_Parser(), args) + + +def test_runtime_agents_rejects_empty_catalog(monkeypatch: pytest.MonkeyPatch) -> None: + """An initialized but empty catalog cannot fall back to the bundled mock pool.""" + monkeypatch.setattr(cli, "PostgresProviderCatalogStore", lambda _dsn: object()) + + class _Service: + def __init__(self, *, store) -> None: + self.store = store + + def candidate_agents(self): + return [] + + monkeypatch.setattr(cli, "ProviderCatalogService", _Service) + args = argparse.Namespace(provider_catalog_dsn="postgresql://catalog", agents="ignored.json") + with pytest.raises(ValueError, match="no enabled candidates"): + cli._runtime_agents(_Parser(), args) + + +def test_main_catalog_mode_uses_provider_aware_client( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Catalog mode wires the provider-aware client into the ordinary orchestrator.""" + _RuntimeOrchestrator.instances.clear() + agent = _catalog_agent() + client_calls: list[dict[str, object]] = [] + monkeypatch.setattr(cli, "_runtime_agents", lambda _parser, _args: [agent]) + monkeypatch.setattr( + cli, + "ProviderAwareModelClient", + lambda **kwargs: client_calls.append(kwargs) or {"provider_client": kwargs}, + ) + monkeypatch.setattr(cli, "TaskOrchestrator", _RuntimeOrchestrator) + monkeypatch.setattr( + sys, + "argv", + [ + "contextual-orchestrator", + "catalog prompt", + "--provider-catalog-dsn", + "postgresql://catalog", + "--mode", + "route", + ], + ) + + cli.main() + + instance = _RuntimeOrchestrator.instances[-1] + assert instance.agents == [agent] + assert instance.kwargs["client"]["provider_client"]["verify_tls"] is True + assert client_calls == [{"ca_bundle": None, "verify_tls": True}] + assert instance.complete_calls == [([{"role": "user", "content": "catalog prompt"}], "route")] + assert json.loads(capsys.readouterr().out) == {"answer": "catalog-answer", "mode": "route"} From 0a1df83a8668fb33c9eb72eeca520a9857d007e6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:46:31 +0900 Subject: [PATCH 07/15] feat: export provider catalog runtime contracts --- contextual_orchestrator/__init__.py | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/contextual_orchestrator/__init__.py b/contextual_orchestrator/__init__.py index 2d4250cd..f22987e3 100644 --- a/contextual_orchestrator/__init__.py +++ b/contextual_orchestrator/__init__.py @@ -46,6 +46,22 @@ get_config_store, ) from .orchestrator import ModelAgent, TaskOrchestrator, WorkflowStep, load_agents +from .provider_catalog import ( + DEFAULT_PROVIDER_ACCOUNTS, + CatalogHttpError, + CatalogModelRecord, + DiscoveredModel, + InMemoryProviderCatalogStore, + PostgresProviderCatalogStore, + ProviderAccount, + ProviderAwareModelClient, + ProviderCatalogHttpClient, + ProviderCatalogService, + ProviderCatalogUnavailable, + bootstrap_provider_credentials, + build_catalog_orchestrator, + normalize_models_document, +) from .token_counting import HeuristicTokenCounter, build_token_counter __all__ = [ @@ -56,6 +72,21 @@ "get_credential", "register_credential", "NotConfigured", + # durable provider catalog + "DEFAULT_PROVIDER_ACCOUNTS", + "ProviderAccount", + "DiscoveredModel", + "CatalogModelRecord", + "CatalogHttpError", + "ProviderCatalogUnavailable", + "InMemoryProviderCatalogStore", + "PostgresProviderCatalogStore", + "ProviderCatalogHttpClient", + "ProviderAwareModelClient", + "ProviderCatalogService", + "bootstrap_provider_credentials", + "normalize_models_document", + "build_catalog_orchestrator", # cost review "ATTRIBUTION_DIMENSIONS", "AttributionDimensions", From 73b1d5085b267ad5ae1ba3bddf2b32b1e13ecc38 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:47:46 +0900 Subject: [PATCH 08/15] docs: specify durable provider catalog architecture --- ...6-08-16-durable-provider-catalog-design.md | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-16-durable-provider-catalog-design.md diff --git a/docs/superpowers/specs/2026-08-16-durable-provider-catalog-design.md b/docs/superpowers/specs/2026-08-16-durable-provider-catalog-design.md new file mode 100644 index 00000000..24c0cf50 --- /dev/null +++ b/docs/superpowers/specs/2026-08-16-durable-provider-catalog-design.md @@ -0,0 +1,187 @@ +# Durable Provider Catalog Design + +## Decision + +`contextual-orchestrator` will treat provider credentials, provider accounts, +model metadata, and orchestration policy as separate control-plane objects. +GitHub Actions secrets are bootstrap transport only. A trusted default-branch +workflow writes the five provider credentials into the existing pgcrypto-backed +credential registry, discovers each account's current model catalog, and stores +normalized model metadata in PostgreSQL. The running gateway reads credential +*names* and model candidates from those durable stores; it does not use provider +API-key environment variables as a runtime source. + +The fixed bootstrap inventory is: + +| Provider account | Credential name | Discovery/transport | +| --- | --- | --- | +| `nvidia_nim_primary` | `NVIDIA_NIM_API_KEY` | OpenAI-compatible `/v1/models` and chat | +| `nvidia_nim_secondary` | `NVIDIA_NIM_API_KEY_SUB` | Independent NIM account, same contract | +| `bytez_primary` | `BYTEZ_API_KEY` | Native Bytez `Key` and `input` contract | +| `openrouter_primary` | `OPENROUTER_API_KEY` | OpenAI-compatible model catalog and chat | +| `openai_primary` | `OPENAI_API_KEY` | OpenAI model catalog and chat | + +NVIDIA's primary and secondary keys remain distinct provider accounts so +quota exhaustion, revocation, health, and circuit state cannot be conflated. + +## Product outcome + +An operator configures the database connection and the five existing Actions +secrets once. The trusted sync job then maintains a candidate pool without +hand-editing an agents JSON file. At service startup, +`--provider-catalog-dsn` replaces the seed file with enabled database models. +The existing paper-grounded route/conduct engine receives the whole role-tagged +pool and continues to decide between one-model routing and a +Thinker–Worker–Verifier–Synthesizer workflow. + +The design does not claim that every listed model is suitable for every task. +Capabilities and modalities constrain routing first; context capacity and +provider/account preference follow; known price is only a small tie-break. The +current deterministic policy remains auditable and replaceable by a learned +router only after evaluation evidence shows that it is the bottleneck. + +## Boundaries + +### Credential plane + +The existing `provider_credentials` table remains the only provider-secret +store. It contains `credential_name` and pgcrypto-encrypted values. The catalog +stores only `credential_name` references. Secret values never appear in model +rows, generated agent JSON, audit summaries, workflow artifacts, or error text. + +`bootstrap_provider_credentials()` validates the complete fixed inventory before +writing when `--require-all` is selected. This prevents a production run from +rotating only a subset and leaving an ambiguous mixed generation. + +### Catalog plane + +The catalog is third-normal-form data: + +- `provider_accounts`: account identity, provider, credential name, endpoint, + transport, enablement, and priority; +- `provider_models`: account-specific model identity, display metadata, context, + known prices, enablement, and first/last observation; +- `model_capabilities`: one capability per model row; +- `model_modalities`: one modality per model row; +- `catalog_refresh_runs`: immutable per-account refresh outcome evidence. + +A successful account refresh atomically upserts the observed set and disables +models absent from that complete response. A failed refresh writes only a +failure record; it never disables or deletes the prior usable set. + +### Discovery plane + +Credentialed catalog HTTP uses HTTPS, direct DNS-resolved public addresses, +normal certificate/SNI verification, no redirect following, no ambient proxy, +a bounded response, strict JSON object validation, bounded attempts, jittered +backoff, and a wall-clock deadline. Authentication and schema errors fail fast. +Transient network, rate-limit, and 5xx errors are isolated to that account. + +The model normalizer accepts the common `data` and `models` shapes, rejects +invalid/oversized identifiers and non-finite metadata, and infers conservative +capabilities from provider metadata plus model naming. Unknown values remain +unknown; the gateway does not fabricate context windows or prices. + +### Inference plane + +OpenAI, OpenRouter, and NVIDIA NIM continue through the hardened +OpenAI-compatible `ModelClient`. Bytez uses `ProviderAwareModelClient` and its +native `Authorization: Key …` plus `{"input": …}` request shape. Unsupported +Bytez passthrough endpoints fail closed instead of pretending that a native +response is an OpenAI Responses or tool-call object. + +Generated `ModelAgent` rows contain model ids, endpoints, provider names, +capabilities, priorities, and credential names only. `TaskOrchestrator` retains +its existing per-agent retry, failover, and circuit-breaker behavior. A provider +catalog with zero enabled candidates is a startup error, not a reason to start a +mock agent. + +## Trusted GitHub Actions flow + +`.github/workflows/provider-catalog-sync.yml` has two trust-separated jobs: + +1. Pull requests run only deterministic offline contracts and compile checks; + provider secrets are not exposed to contributor code. +2. Scheduled/manual runs execute only on protected `main` in the `production` + environment. They require the five provider keys plus + `CONTEXTUAL_ORCHESTRATOR_KV_DSN` and + `CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE`, seed the encrypted registry, refresh + metadata, and verify that generated agent evidence contains no secret value. + +Missing database bootstrap secrets block the job. The workflow never downgrades +to process memory, because an ephemeral registry would create a false success +and disappear before the service could use it. + +## Failure semantics + +| Condition | Result | Operator action | +| --- | --- | --- | +| One provider catalog is unavailable and has prior models | Serve prior models as `stale_available`; refresh peers | Inspect provider health; retry next schedule | +| One provider is unavailable with no prior models | Mark account `failed`; continue peers | Correct endpoint/key or wait for provider | +| All providers fail and no prior model exists | Fail sync/startup | Restore DB/provider connectivity before service | +| Credential missing | Account failure; production `--require-all` blocks before writes | Add/repair the named Actions secret | +| 401/403 | Permanent account failure, no retry storm | Rotate/re-authorize that credential | +| 408/429/5xx/network timeout | Bounded jittered retry, then stale/failure classification | Observe rate and provider SLO | +| Invalid/oversized/non-JSON response | Fail closed without body disclosure | Treat as provider contract/security incident | +| Database failure | Fail closed; no memory fallback | Restore the authoritative catalog/KV database | +| Bytez unsupported response/passthrough | Fail closed, allow normal orchestrator failover where available | Use a supported native chat model or another provider | + +## Test and acceptance evidence + +The feature is accepted only when all of the following hold on one exact PR +head: + +- all five fixed credential names are represented and NVIDIA accounts remain + independent; +- required bootstrap is all-or-nothing and summaries contain no value; +- normalization, malformed metadata, specialized capabilities, and price/context + bounds are deterministic; +- provider failures are isolated and last-known-good models survive; +- no-candidate startup fails closed; +- discovered models become valid two-or-more-word snake-case agents; +- role selection and cross-provider failover use the complete candidate pool; +- native Bytez authentication/response handling is tested independently; +- PostgreSQL DDL is normalized and contains no secret-value column; +- the full repository test, 100% branch coverage, 100% public docstring, + security, fuzz, and protected review gates pass without weakening them; and +- the protected default-branch sync subsequently records real provider and DB + evidence without revealing credentials. + +## Research and standards basis + +The route/conduct split follows the repository's existing Fugu, TRINITY, and +Conductor interpretation: cheap single-model selection for suitable work, deeper +role-separated computation when decomposition and verification add value. The +catalog makes the swappable model pool operational rather than static. Cost is +kept subordinate to capability, consistent with cost-aware routing literature +that optimizes under quality constraints rather than choosing the cheapest model +unconditionally. + +### References + +Chen, L., Zaharia, M., & Zou, J. (2023). *FrugalGPT: How to use large language +models while reducing cost and improving performance*. arXiv. +https://doi.org/10.48550/arXiv.2305.05176 + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). +Internet Engineering Task Force. https://doi.org/10.17487/RFC9110 + +*Learning to orchestrate agents in natural language with the Conductor*. +(2025). arXiv. https://arxiv.org/abs/2512.04388 + +National Institute of Standards and Technology. (2023). *Artificial intelligence +risk management framework (AI RMF 1.0)* (NIST AI 100-1). +https://doi.org/10.6028/NIST.AI.100-1 + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., Kadous, +M. W., & Stoica, I. (2024). *RouteLLM: Learning to route LLMs with preference +data*. arXiv. https://doi.org/10.48550/arXiv.2406.18665 + +PostgreSQL Global Development Group. (2026). *pgcrypto*. +https://www.postgresql.org/docs/current/pgcrypto.html + +Sakana AI. (2026). *Fugu technical report*. +https://github.com/SakanaAI/fugu/blob/main/Fugu_technical_report.pdf + +*TRINITY: An evolved LLM coordinator*. (2025). arXiv. +https://arxiv.org/abs/2512.04695 From 3f28b68513192b0a3eb7c698b600be1b30832f3a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:48:42 +0900 Subject: [PATCH 09/15] docs: add provider catalog implementation plan --- .../2026-08-16-durable-provider-catalog.md | 276 ++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-16-durable-provider-catalog.md diff --git a/docs/superpowers/plans/2026-08-16-durable-provider-catalog.md b/docs/superpowers/plans/2026-08-16-durable-provider-catalog.md new file mode 100644 index 00000000..39069365 --- /dev/null +++ b/docs/superpowers/plans/2026-08-16-durable-provider-catalog.md @@ -0,0 +1,276 @@ +# Durable Provider Catalog Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Persist the five organization provider credentials and normalized model catalogs, then start `contextual-orchestrator` from an automatically discovered, role-tagged, multi-provider agent pool. + +**Architecture:** A trusted default-branch workflow seeds the existing encrypted credential registry and refreshes a normalized PostgreSQL provider catalog account by account. Runtime startup loads enabled catalog rows into ordinary `ModelAgent` records and uses the existing route/conduct engine plus a narrow native Bytez transport. Failures remain provider-scoped when last-known-good data exists and fail closed when no usable candidate remains. + +**Tech Stack:** Python 3.10+, standard-library HTTP/TLS, PostgreSQL + pgcrypto/psycopg optional DB extra, pytest/Hypothesis-compatible deterministic tests, GitHub Actions. + +## Global Constraints + +- Runtime provider keys resolve from the credential registry, never directly from environment variables. +- GitHub Actions environment variables are bootstrap transport only. +- Fixed credentials: `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, `BYTEZ_API_KEY`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`. +- Production durable bootstrap also requires `CONTEXTUAL_ORCHESTRATOR_KV_DSN` and `CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE`. +- Database objects use two-or-more-word snake_case and third normal form. +- Capability and role fit outrank known price; price is a bounded tie-break. +- Provider catalog/network/database errors never include credential values or raw provider bodies. +- Exact-head repository coverage and public-docstring coverage remain 100%. +- Existing security, fuzz, review, and branch-protection gates may not be weakened. + +--- + +### Task 1: Lock the provider inventory and normalization contracts + +**Files:** +- Create: `tests/test_provider_catalog.py` +- Create: `tests/test_provider_catalog_coverage.py` +- Create: `contextual_orchestrator/provider_catalog.py` + +**Interfaces:** +- Produces: `ProviderAccount`, `DiscoveredModel`, `CatalogModelRecord`, `DEFAULT_PROVIDER_ACCOUNTS`, `normalize_models_document(document) -> list[DiscoveredModel]`. +- Consumes: `register_credential`, `get_credential`, `ModelAgent`. + +- [x] **Step 1: Write failing inventory and normalization tests** + +```python +def test_default_accounts_cover_every_configured_secret_and_split_nvidia_accounts(): + assert [row.credential_name for row in DEFAULT_PROVIDER_ACCOUNTS] == [ + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "BYTEZ_API_KEY", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", + ] +``` + +- [x] **Step 2: Verify RED** + +Run: + +```bash +python -m pytest tests/test_provider_catalog.py -q +``` + +Expected before implementation: import failure for `contextual_orchestrator.provider_catalog`. + +- [x] **Step 3: Implement the fixed accounts and provider-neutral normalizer** + +Implement bounded ids, contexts, prices, modalities, and conservative capability inference. Keep unknown values `None`; never fabricate a price or context window. + +- [x] **Step 4: Verify GREEN** + +```bash +python -m pytest tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py -q +``` + +Expected: inventory and normalization contracts pass. + +- [x] **Step 5: Commit** + +```bash +git add tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py contextual_orchestrator/provider_catalog.py +git commit -m "feat: add durable multi-provider model catalog" +``` + +### Task 2: Add isolated refresh and normalized persistence + +**Files:** +- Modify: `contextual_orchestrator/provider_catalog.py` +- Test: `tests/test_provider_catalog.py` +- Test: `tests/test_provider_catalog_coverage.py` + +**Interfaces:** +- Produces: `ProviderCatalogStore`, `InMemoryProviderCatalogStore`, `PostgresProviderCatalogStore`, `ProviderCatalogService.refresh_all()`, `PROVIDER_CATALOG_SCHEMA_SQL`. +- Consumes: provider accounts and normalized models from Task 1. + +- [x] **Step 1: Write failing last-known-good and no-candidate tests** + +```python +def test_refresh_isolates_provider_failure_and_preserves_last_known_good_catalog(): + # Seed two accounts; fail one refresh; assert its old model remains and peer updates. + ... +``` + +- [x] **Step 2: Verify RED** + +```bash +python -m pytest tests/test_provider_catalog.py -k refresh -q +``` + +Expected before implementation: missing service/store methods. + +- [x] **Step 3: Implement account-scoped transactions and refresh evidence** + +Use `provider_accounts`, `provider_models`, `model_capabilities`, `model_modalities`, and `catalog_refresh_runs`. Disable missing models only after a complete successful account refresh. A failed refresh inserts failure evidence and leaves prior model rows untouched. + +- [x] **Step 4: Verify GREEN and schema rules** + +```bash +python -m pytest tests/test_provider_catalog.py -k "refresh or schema" -q +``` + +Expected: isolated refresh, stale availability, empty-catalog failure, and no-secret schema tests pass. + +- [x] **Step 5: Commit** + +```bash +git add contextual_orchestrator/provider_catalog.py tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py +git commit -m "feat: persist normalized provider catalogs" +``` + +### Task 3: Build the automatic runtime pool and native Bytez seam + +**Files:** +- Modify: `contextual_orchestrator/provider_catalog.py` +- Modify: `contextual_orchestrator/__init__.py` +- Modify: `contextual_orchestrator/__main__.py` +- Create: `tests/test_provider_catalog_cli.py` +- Test: `tests/test_provider_catalog.py` + +**Interfaces:** +- Produces: `ProviderCatalogService.candidate_agents()`, `ProviderAwareModelClient`, `build_catalog_orchestrator()`, CLI `--provider-catalog-dsn`. +- Consumes: `TaskOrchestrator`, `ModelClient`, enabled catalog rows, KV credential names. + +- [x] **Step 1: Write failing role-routing, failover, Bytez, and CLI tests** + +```python +def test_catalog_orchestrator_uses_role_tags_and_retains_cross_provider_failover(): + orchestrator = build_catalog_orchestrator(store, accounts=(reasoning, coding)) + assert orchestrator._select_agent("plan", "thinker").model == "deep-reasoner" + assert orchestrator._select_agent("implement code", "worker").model == "code-specialist" +``` + +- [x] **Step 2: Verify RED** + +```bash +python -m pytest tests/test_provider_catalog.py tests/test_provider_catalog_cli.py -q +``` + +Expected before implementation: missing catalog factory/client/CLI option. + +- [x] **Step 3: Implement agent conversion and provider-aware transport** + +Generate bounded two-or-more-word snake-case ids. Map chat/reasoning/coding/vision/audio capabilities into existing role tags. Keep role fit ahead of context and price. Delegate OpenAI-compatible providers to the existing secure client; use native Bytez `Key` plus `input` only for ordinary Bytez chat and fail closed for unsupported passthrough shapes. + +- [x] **Step 4: Verify GREEN** + +```bash +python -m pytest tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py tests/test_provider_catalog_cli.py -q +``` + +Expected: role selection, two-account NIM failover, Bytez native output, and catalog CLI startup pass. + +- [x] **Step 5: Commit** + +```bash +git add contextual_orchestrator/provider_catalog.py contextual_orchestrator/__init__.py contextual_orchestrator/__main__.py tests/test_provider_catalog*.py +git commit -m "feat: start runtime from discovered provider models" +``` + +### Task 4: Add the trust-separated GitHub Actions bootstrap + +**Files:** +- Create: `.github/workflows/provider-catalog-sync.yml` +- Test: `tests/test_provider_catalog.py` + +**Interfaces:** +- Produces: pull-request offline contract job and protected-main credential/catalog synchronization job. +- Consumes: fixed provider secrets, durable KV DSN/passphrase, module CLI `bootstrap-and-sync`. + +- [x] **Step 1: Encode the untrusted/trusted job boundary** + +Pull requests receive no provider or database secrets. Scheduled/manual execution is restricted to `refs/heads/main` and the protected `production` environment. + +- [x] **Step 2: Add complete-inventory validation** + +```bash +required=( + CONTEXTUAL_ORCHESTRATOR_KV_DSN + CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE + NVIDIA_NIM_API_KEY NVIDIA_NIM_API_KEY_SUB BYTEZ_API_KEY OPENROUTER_API_KEY OPENAI_API_KEY +) +``` + +Fail before bootstrap when any value is empty; add each configured value to Actions masking without printing it. + +- [x] **Step 3: Seed, refresh, and verify secret-free evidence** + +```bash +python -m contextual_orchestrator.provider_catalog bootstrap-and-sync \ + --require-all --agents-output "$RUNNER_TEMP/provider-agents.json" +``` + +Parse the generated agent pool and safe summary; fail if no candidate exists or any secret value appears. + +- [x] **Step 4: Validate workflow syntax and offline contracts** + +```bash +python -m pytest tests/test_provider_catalog.py tests/test_provider_catalog_coverage.py -q +python -m compileall -q contextual_orchestrator +``` + +- [x] **Step 5: Commit** + +```bash +git add .github/workflows/provider-catalog-sync.yml +git commit -m "ci: add trusted provider catalog bootstrap" +``` + +### Task 5: Ground, document, and verify the exact head + +**Files:** +- Create: `docs/superpowers/specs/2026-08-16-durable-provider-catalog-design.md` +- Create: `docs/superpowers/plans/2026-08-16-durable-provider-catalog.md` +- Create: `docs/doctoring/durable-provider-catalog.md` +- Create: `docs/provider_catalog.md` +- Modify: `CHANGELOG.md` + +**Interfaces:** +- Produces: operator recovery/rollback instructions, APA 7 source record, release note. +- Consumes: implementation and workflow behavior from Tasks 1–4. + +- [x] **Step 1: Document architecture and operational actions** + +State credential/catalog separation, 3NF objects, refresh semantics, Bytez native boundary, startup behavior, exact secret inventory, and required DB bootstrap secrets. + +- [x] **Step 2: Add APA 7 doctoring** + +Ground the design in FrugalGPT, RouteLLM, Fugu/TRINITY/Conductor, HTTP semantics, NIST AI RMF, and PostgreSQL pgcrypto. Do not attach a PDF unless redistribution is permitted; reuse the repository's existing OA routing PDFs. + +- [ ] **Step 3: Run exact full verification** + +```bash +python -m coverage erase +python -m coverage run --branch -m pytest -q +python -m coverage report --fail-under=100 +interrogate --fail-under 100 contextual_orchestrator +python -m compileall -q contextual_orchestrator +python -m pip check +git diff --check +``` + +Expected: zero failures, 100% measured branch coverage, 100% public-docstring coverage, no dependency conflict, and no whitespace errors. + +- [ ] **Step 4: Run repository security gates** + +```bash +trivy --download-db-only +trivy fs --severity CRITICAL,HIGH --ignore-unfixed . +python -m pip_audit -r requirements.lock +``` + +Expected: no unremediated high/critical finding. Do not weaken a gate. + +- [ ] **Step 5: Publish the stacked PR and wait for protected evidence** + +```bash +git push -u origin feature/durable-provider-catalog-v2 +gh pr create --base fix/atheris-interpreter-lock \ + --head feature/durable-provider-catalog-v2 \ + --title "feat: durable automatic multi-provider catalog" +``` + +Target the accepted provider-security branch so DNS-pinned/strict-response work is inherited before protected `main`. Require all exact-head checks, current reviews, zero unresolved valid findings, and qualifying independent approval before normal merge. From 29f876e7136ab5892cd51961b8b84d4a6b36f760 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:49:42 +0900 Subject: [PATCH 10/15] docs: add provider catalog doctoring and recovery --- docs/doctoring/durable-provider-catalog.md | 221 +++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 docs/doctoring/durable-provider-catalog.md diff --git a/docs/doctoring/durable-provider-catalog.md b/docs/doctoring/durable-provider-catalog.md new file mode 100644 index 00000000..3ffa92b1 --- /dev/null +++ b/docs/doctoring/durable-provider-catalog.md @@ -0,0 +1,221 @@ +# Durable Provider Catalog Doctoring + +## Purpose + +This record explains why provider credentials and model catalogs are separate, +how the five configured provider accounts become an orchestration pool, which +failures are tolerated, and which failures stop service. It is the operational +source for incident response, rollback, and audit review. + +## Invariants + +1. Provider API-key values exist only in the encrypted credential registry. +2. The provider catalog contains credential names, never secret values. +3. `NVIDIA_NIM_API_KEY` and `NVIDIA_NIM_API_KEY_SUB` are independent accounts. +4. Pull-request code never receives production provider or database secrets. +5. A configured PostgreSQL catalog/KV is authoritative; failure cannot silently + downgrade to process memory. +6. A failed provider refresh cannot disable its last-known-good models. +7. A complete successful refresh may disable models absent from that account's + new complete listing. +8. Zero usable candidates is a startup/sync failure, not permission to use mocks. +9. Capability and role fit outrank context and cost; price is a bounded tie-break. +10. Native Bytez requests use its Key/input contract; unsupported OpenAI + passthrough shapes fail closed. + +## Bootstrap sequence + +The trusted protected-default-branch workflow performs these actions: + +1. Require non-empty `CONTEXTUAL_ORCHESTRATOR_KV_DSN`, + `CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE`, and all five provider keys. +2. Add values to GitHub Actions masking without echoing them. +3. Select `CONTEXTUAL_ORCHESTRATOR_KV_BACKEND=postgres`. +4. Validate the complete fixed inventory before any provider credential write. +5. Upsert credentials through `register_credential()` into pgcrypto storage. +6. Refresh each provider account independently over bounded credentialed HTTPS. +7. Upsert normalized provider/model/capability/modality rows transactionally. +8. Preserve prior rows for failed accounts and classify them `stale_available`. +9. Generate a secret-free agent pool and reject zero candidates. +10. Inspect the safe summary and generated JSON for any exact secret value. + +Do not copy a provider key into `--agents`, repository variables, command-line +arguments, artifacts, cache keys, logs, issue comments, or deployment manifests. + +## Runtime sequence + +Start the gateway with the durable catalog connection: + +```bash +python -m contextual_orchestrator --serve \ + --provider-catalog-dsn "$CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN" \ + --admin-token "$CONTEXTUAL_ORCHESTRATOR_ADMIN_TOKEN" \ + --inference-token "$CONTEXTUAL_ORCHESTRATOR_INFERENCE_TOKEN" +``` + +The DSN connects to the catalog; it is not a provider API key. Startup loads only +enabled account/model rows. Each `ModelAgent` carries a credential name, and the +provider client resolves the current value from the credential registry at the +request boundary. Credential rotation therefore does not require rewriting +model rows. + +The existing orchestration engine receives the complete pool. Fast route mode +selects one model. Conduct mode selects role-appropriate Thinker, Worker, +Verifier, and Synthesizer candidates and retains other eligible accounts as +failover. Provider retries remain bounded and circuit breakers prevent a +persistently failing account from being selected continuously. + +## Exception matrix + +| Failure | Retry | Catalog mutation | Service effect | Required action | +| --- | --- | --- | --- | --- | +| DNS, connect, timeout | Bounded jitter | Failure row only | Stale models continue if present | Check egress/DNS/provider status | +| HTTP 408/409/425/429/5xx | Bounded jitter | Failure row only after exhaustion | Account stale/failed; peers continue | Inspect rate limits and provider SLO | +| HTTP 401/403 | No retry storm | Failure row only | Account stale/failed | Rotate or reauthorize named key | +| Redirect | Reject | Failure row only | Account stale/failed | Correct canonical endpoint; do not follow credential redirects | +| Private/reserved destination | Reject before credential send | Failure row only | Account stale/failed | Treat as SSRF/configuration incident | +| Non-JSON, duplicate/invalid JSON, excessive body | Reject | Failure row only | Account stale/failed | Treat as provider contract/security incident | +| Missing key in required bootstrap | No writes | None | Whole production bootstrap blocked | Configure the exact Actions secret | +| Missing key in optional local bootstrap | No write for account | Failure row on sync | Peers may continue | Seed key before production | +| PostgreSQL unavailable | No memory fallback | None | Sync/startup blocked | Restore authoritative database | +| Empty successful listing | No destructive replacement | Failure row only | Prior models stay; otherwise account failed | Verify provider list entitlement/contract | +| Bytez unsupported output | No repair/guess | Runtime failure only | Orchestrator may use another eligible account | Select supported native model/adapter | +| Bytez tool/Responses passthrough | Reject | None | Request fails closed | Route that contract to an OpenAI-compatible candidate | +| All accounts unavailable, no prior model | Bounded account attempts | Failure evidence where DB works | Gateway does not start | Restore at least one validated provider | + +Raw exception messages and response bodies are not public error contracts because +they can contain provider-controlled or sensitive content. Stable reason codes +are the operational interface. + +## Rotation procedure + +1. Add the new key value to the existing Actions secret name. +2. Manually run **Provider Catalog Sync** on protected `main` in the production + environment. +3. Confirm the safe summary reports the credential name and at least one model. +4. Confirm no account unexpectedly changed to `failed` or `stale_available`. +5. Send a bounded canary inference through that account. +6. Revoke the old provider key only after the canary succeeds. +7. Confirm the next scheduled refresh and runtime call resolve the new value. + +The database upsert replaces the encrypted value under the same credential name; +model rows and consuming services require no secret-bearing change. + +## Incident response + +### Suspected credential disclosure + +- Revoke/rotate the provider key immediately. +- Run trusted bootstrap to replace the encrypted registry value. +- Inspect Actions, application, proxy, database-audit, and provider logs for the + credential name and access time; do not paste the value into searches or tickets. +- Verify generated agent JSON and workflow summaries remain value-free. +- Treat a provider-side unauthorized model invocation as a security incident. + +### Catalog poisoning or malformed listing + +- Disable the affected `provider_accounts.enabled_flag` row. +- Preserve the response only in an access-controlled incident store; do not add + it to public CI logs. +- Confirm other providers still supply role coverage. +- Reproduce with a sanitized fixture and add a failing parser/transport test. +- Re-enable only after exact-head security tests and a clean refresh. + +### Database outage + +- Do not select memory mode as an automatic recovery mechanism. +- Restore the authoritative PostgreSQL service, network path, and pgcrypto + passphrase access. +- Validate `provider_credentials`, `provider_accounts`, `provider_models`, and the + latest `catalog_refresh_runs` before restarting the gateway. +- If emergency local mock service is intentionally required, start it as an + explicitly separate non-production deployment and label all evidence accordingly. + +## Rollback + +Code rollback may remove `--provider-catalog-dsn` and return a deployment to an +explicit reviewed agents file, but it must not copy API-key values into that +file or reintroduce runtime environment lookup. Keep the credential registry and +catalog tables during rollback; they are backward-compatible control-plane data +and preserve audit evidence. + +A schema rollback is normally unnecessary. If required, first export account, +model, capability, modality, and refresh metadata without secret values. Drop +catalog tables only after all catalog-backed services are stopped. Do not drop +`provider_credentials` as part of a model-catalog rollback. + +## Verification commands + +```bash +python -m pytest \ + tests/test_provider_catalog.py \ + tests/test_provider_catalog_coverage.py \ + tests/test_provider_catalog_cli.py -q +python -m coverage erase +python -m coverage run --branch -m pytest -q +python -m coverage report --fail-under=100 +interrogate --fail-under 100 contextual_orchestrator +python -m compileall -q contextual_orchestrator +python -m pip check +git diff --check +``` + +The trusted live sync is separate evidence. Pull-request success proves parser, +store, routing, failure, and workflow contracts without proving that any current +provider credential or production database is healthy. + +## Evidence interpretation + +- `refreshed`: current provider listing committed successfully. +- `stale_available`: current refresh failed, but prior enabled models remain. +- `failed`: refresh failed and that account has no usable prior model. +- `disabled`: governance deliberately excluded the account. +- `candidate_model_count`: enabled account/model pairs, not a quality claim. +- inferred capability: routing hint derived conservatively from metadata/name, + not a provider guarantee or benchmark result. +- price: stored only when supplied and finite; absent is `NULL`, not zero. + +## Research and standards rationale + +FrugalGPT and RouteLLM show that model selection can improve the +quality–cost frontier, but only when routing respects task quality rather than +using price alone. Fugu, TRINITY, and Conductor motivate a swappable model pool, +role assignment, selective context, and a route-versus-deep-orchestration split. +The durable catalog supplies that pool while retaining an auditable deterministic +policy until learned routing has a valid evaluation set. + +RFC 9110 informs retry and status classification: safe catalog GET operations may +be retried within explicit limits, while authentication failures and ambiguous +contracts fail fast. NIST AI RMF supports traceable inventory, monitoring, and +risk treatment. PostgreSQL pgcrypto provides the existing encryption-at-rest +boundary, while table separation prevents model metadata queries from exposing +secret values. + +## References + +Chen, L., Zaharia, M., & Zou, J. (2023). *FrugalGPT: How to use large language +models while reducing cost and improving performance*. arXiv. +https://doi.org/10.48550/arXiv.2305.05176 + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). +Internet Engineering Task Force. https://doi.org/10.17487/RFC9110 + +*Learning to orchestrate agents in natural language with the Conductor*. +(2025). arXiv. https://arxiv.org/abs/2512.04388 + +National Institute of Standards and Technology. (2023). *Artificial intelligence +risk management framework (AI RMF 1.0)* (NIST AI 100-1). +https://doi.org/10.6028/NIST.AI.100-1 + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., Kadous, +M. W., & Stoica, I. (2024). *RouteLLM: Learning to route LLMs with preference +data*. arXiv. https://doi.org/10.48550/arXiv.2406.18665 + +PostgreSQL Global Development Group. (2026). *pgcrypto*. +https://www.postgresql.org/docs/current/pgcrypto.html + +Sakana AI. (2026). *Fugu technical report*. +https://github.com/SakanaAI/fugu/blob/main/Fugu_technical_report.pdf + +*TRINITY: An evolved LLM coordinator*. (2025). arXiv. +https://arxiv.org/abs/2512.04695 From 6ce0f14ea13745af3ceabe6932fc04e3caa3591a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:50:14 +0900 Subject: [PATCH 11/15] docs: add provider catalog operator guide --- docs/provider_catalog.md | 129 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 docs/provider_catalog.md diff --git a/docs/provider_catalog.md b/docs/provider_catalog.md new file mode 100644 index 00000000..6ec8a38c --- /dev/null +++ b/docs/provider_catalog.md @@ -0,0 +1,129 @@ +# Provider catalog operator guide + +Use this guide to turn the five existing organization provider secrets into the +runtime model pool without placing API-key values in source, agent JSON, or the +long-running process environment. + +## Required Actions secrets + +Provider credentials: + +- `NVIDIA_NIM_API_KEY` +- `NVIDIA_NIM_API_KEY_SUB` +- `BYTEZ_API_KEY` +- `OPENROUTER_API_KEY` +- `OPENAI_API_KEY` + +Durable registry/catalog bootstrap: + +- `CONTEXTUAL_ORCHESTRATOR_KV_DSN` +- `CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE` + +The provider keys already named above are not sufficient by themselves to create +a durable database result. The DSN and passphrase tell the trusted job where the +pgcrypto registry/catalog lives and how to decrypt credentials later. When either +is absent, the workflow fails instead of reporting success against temporary +memory. + +## First synchronization + +After the feature reaches protected `main`: + +1. Open **Actions → Provider Catalog Sync → Run workflow**. +2. Select protected `main` and the `production` environment. +3. Wait for **Seed credentials and refresh durable catalog** to finish. +4. Read only the safe summary: + - `candidate_agent_count` must be greater than zero; + - each intended account should be `refreshed`; + - `stale_available` is serviceable but requires provider investigation; + - `failed` means that account has no usable discovered model. +5. Do not copy a provider key into an issue when diagnosing a failure. Use the + credential name and stable error code. + +The same workflow runs every six hours. Each provider refresh is isolated, so one +outage does not erase other accounts or its own last-known-good models. + +## Start the gateway from the catalog + +```bash +export CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN="$CONTEXTUAL_ORCHESTRATOR_KV_DSN" +python -m contextual_orchestrator --serve \ + --provider-catalog-dsn "$CONTEXTUAL_ORCHESTRATOR_CATALOG_DSN" \ + --admin-token "$CONTEXTUAL_ORCHESTRATOR_ADMIN_TOKEN" \ + --inference-token "$CONTEXTUAL_ORCHESTRATOR_INFERENCE_TOKEN" \ + --host 127.0.0.1 \ + --port 8000 +``` + +`--provider-catalog-dsn` is authoritative. It disables the seed agents file and +loads enabled database models. Startup fails when the database is unavailable or +contains no enabled candidate; it does not silently start `examples/agents.mock.json`. + +OpenAI, OpenRouter, and NVIDIA NIM models use the hardened OpenAI-compatible +transport. Bytez models use the native Bytez adapter. A Bytez request that needs +an unsupported Responses/tool passthrough fails closed rather than returning a +fabricated OpenAI object; another eligible provider should be selected for that +contract. + +## Confirm the pool + +Use the authenticated admin agent-pool endpoint or console and verify: + +- provider names include the accounts refreshed successfully; +- NVIDIA primary and secondary entries have different agent ids and credential + names; +- no agent JSON contains a provider key value; +- reasoning, coding, vision, audio, and embedding models carry only capabilities + supported or conservatively inferred from catalog metadata; +- unknown context and price fields remain absent/null rather than zero; +- disabled accounts are absent from runtime candidates but remain in catalog + history. + +## Respond to common failures + +### `provider credential inventory is incomplete` + +Add or repair the exact missing Actions secret, then rerun the protected workflow. +The required bootstrap performs no partial credential write. + +### `provider catalog requires a PostgreSQL DSN` + +Configure `CONTEXTUAL_ORCHESTRATOR_KV_DSN`. Do not replace it with a temporary +SQLite or memory path in production. + +### `catalog_authentication_failed` + +Rotate or reauthorize the named provider credential. The catalog client does not +retry 401/403 repeatedly. + +### `stale_available` + +The current refresh failed, but the last complete catalog remains enabled. Check +provider status, egress, entitlement, and rate limits. The next scheduled job +will retry within bounded limits. + +### `no usable provider model exists after catalog refresh` + +All enabled accounts lack both a current and prior candidate. Restore at least +one provider or the database before starting the gateway. Do not bypass this by +starting an unlabeled mock deployment. + +### Bytez response/passthrough failure + +Confirm the selected model supports ordinary native chat input. Route OpenAI +Responses, tool calling, or structured passthrough to a provider whose contract +supports it. Do not add response-shape guessing. + +## Rotation without downtime + +1. Replace the value under the existing Actions secret name. +2. Run Provider Catalog Sync manually. +3. Verify the affected account refreshes and a canary succeeds. +4. Revoke the old key. +5. Confirm the next runtime request resolves the updated registry value. + +The model catalog refers to the stable credential name, so no model-row or +consumer configuration change is needed during rotation. + +For incident handling, rollback, and evidence interpretation, read +[`docs/doctoring/durable-provider-catalog.md`](doctoring/durable-provider-catalog.md). From d9fa97eb00b59ce940892403658eb63f5e8354a2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:50:47 +0900 Subject: [PATCH 12/15] docs: record automatic provider catalog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a04eea2..ba9c79da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and ## [Unreleased] +### Added + +- Add a durable, normalized provider catalog for the organization `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, `BYTEZ_API_KEY`, `OPENROUTER_API_KEY`, and `OPENAI_API_KEY` accounts; trusted bootstrap writes values only to the encrypted credential registry, discovers provider models account by account, preserves last-known-good catalogs on isolated failures, generates role-tagged agents, and starts the gateway from enabled database candidates with `--provider-catalog-dsn`. +- Add a provider-aware runtime client that preserves the hardened OpenAI-compatible transport for OpenAI, OpenRouter, and NVIDIA NIM while using a narrow native Bytez Key/input adapter and failing closed for unsupported Bytez passthrough response shapes. +- Add a trust-separated Provider Catalog Sync workflow: pull requests run secret-free offline contracts, while protected-main scheduled/manual runs require the complete five-key inventory plus durable KV DSN/passphrase, verify generated evidence contains no secret value, and never downgrade a configured database to process memory. + ### Security - Fail closed with a stable redacted error when an explicitly configured Postgres KV backend cannot be imported, initialized, or seeded, and route `CostRoutingCoordinator(postgres_dsn=...)` through that authoritative factory, preventing a silent downgrade of configuration, routing, price, and credential authority to process-local memory. @@ -31,6 +37,7 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and ### Documentation +- Add durable provider-catalog design, implementation plan, operator guide, and APA 7 doctoring covering credential/catalog separation, normalized data, account-isolated refresh, route/conduct pool construction, native Bytez handling, trusted Actions bootstrap, rotation, incident response, evidence interpretation, and rollback. - Add APA 7 doctoring for Python environment-marker semantics, Atheris artifact availability and hashes, and the supported-platform uncertainty boundary. - Add provider-response resource-bound doctoring covering the 8 MiB fail-closed limit, HTTP framing preflight, `text/event-stream` media-type enforcement, bounded SSE reads, OpenAI-compatible `[DONE]` completion evidence, malformed-event and premature-EOF handling, batch-output partitioning, incident handling, and operational rollback. - Add provider-stream UTF-8 doctoring grounding strict SSE/JSON decoding and redacted malformed-input handling in the WHATWG HTML Standard and RFC 8259, with verification, failure, rollback, and authority boundaries. From 33484ded9eae77c0dd208965b5be732a40483d0b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:51:19 +0900 Subject: [PATCH 13/15] docs: extend 3NF provider catalog schema --- docs/database_design.sql | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docs/database_design.sql b/docs/database_design.sql index fb589247..90ab301f 100644 --- a/docs/database_design.sql +++ b/docs/database_design.sql @@ -10,6 +10,63 @@ create table agent_pool ( updated_at timestamptz not null default now() ); +create table provider_accounts ( + provider_account_id text primary key, + provider_name text not null, + credential_name text not null, + base_url text not null, + models_path text, + transport_name text not null, + auth_header_name text not null, + auth_prefix text not null, + enabled_flag boolean not null default true, + priority_rank integer not null default 0, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +create table provider_models ( + provider_model_id text primary key, + provider_account_id text not null references provider_accounts(provider_account_id), + model_name text not null, + display_name text not null, + context_window integer, + input_price_usd_per_million numeric(20, 8), + output_price_usd_per_million numeric(20, 8), + enabled_flag boolean not null default true, + first_discovered_at timestamptz not null, + last_seen_at timestamptz not null, + unique (provider_account_id, model_name) +); + +create table model_capabilities ( + provider_model_id text not null references provider_models(provider_model_id) on delete cascade, + capability_name text not null, + primary key (provider_model_id, capability_name) +); + +create table model_modalities ( + provider_model_id text not null references provider_models(provider_model_id) on delete cascade, + modality_name text not null, + primary key (provider_model_id, modality_name) +); + +create table catalog_refresh_runs ( + catalog_refresh_id text primary key, + provider_account_id text not null references provider_accounts(provider_account_id), + refresh_status text not null, + observed_model_count integer not null default 0, + error_code text, + started_at timestamptz not null, + finished_at timestamptz not null +); + +create index provider_models_account_idx + on provider_models (provider_account_id, enabled_flag); + +create index catalog_refresh_account_idx + on catalog_refresh_runs (provider_account_id, finished_at desc); + create table orchestration_policy ( policy_id text primary key, policy_name text not null, From 0264f1d7ac7938a72ac3590deb3c1a98ef76921b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:51:58 +0900 Subject: [PATCH 14/15] test: cover provider catalog terminal retry edges --- tests/test_provider_catalog_edge_cases.py | 48 +++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/test_provider_catalog_edge_cases.py diff --git a/tests/test_provider_catalog_edge_cases.py b/tests/test_provider_catalog_edge_cases.py new file mode 100644 index 00000000..bc996769 --- /dev/null +++ b/tests/test_provider_catalog_edge_cases.py @@ -0,0 +1,48 @@ +"""Terminal retry and metadata edge cases for the provider catalog.""" + +from __future__ import annotations + +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.provider_catalog import ( # noqa: E402 + DEFAULT_PROVIDER_ACCOUNTS, + CatalogHttpError, + ProviderCatalogHttpClient, + normalize_models_document, +) + + +def test_terminal_transient_error_is_not_slept_or_retried() -> None: + """A one-attempt policy surfaces its stable transient code immediately.""" + sleeps: list[float] = [] + client = ProviderCatalogHttpClient(max_attempts=1, sleep=sleeps.append) + client._request_json = lambda _account, _credential: (_ for _ in ()).throw( # type: ignore[method-assign] + CatalogHttpError("catalog_http_503", transient=True) + ) + with pytest.raises(CatalogHttpError, match="catalog_http_503"): + client.discover(DEFAULT_PROVIDER_ACCOUNTS[0], "credential") + assert sleeps == [] + + +def test_boolean_context_and_empty_display_name_are_bounded() -> None: + """Boolean context metadata is rejected and empty display names fall back to ids.""" + model = normalize_models_document( + { + "data": [ + { + "id": "fallback-model", + "name": " ", + "context_length": True, + "pricing": {"prompt": False}, + } + ] + } + )[0] + assert model.display_name == "fallback-model" + assert model.context_window is None + assert model.input_price_usd_per_million is None From 64dc117b1133b79fd970fd61dcc41e116715dc2c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:58:37 +0900 Subject: [PATCH 15/15] test: cover native provider output variants --- tests/test_provider_catalog_output_shapes.py | 73 ++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 tests/test_provider_catalog_output_shapes.py diff --git a/tests/test_provider_catalog_output_shapes.py b/tests/test_provider_catalog_output_shapes.py new file mode 100644 index 00000000..a74c394a --- /dev/null +++ b/tests/test_provider_catalog_output_shapes.py @@ -0,0 +1,73 @@ +"""Native provider output variants and safe summary contracts.""" + +from __future__ import annotations + +import json +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + set_backend, +) +from contextual_orchestrator.orchestrator import ModelAgent # noqa: E402 +from contextual_orchestrator.provider_catalog import ( # noqa: E402 + DEFAULT_PROVIDER_ACCOUNTS, + ProviderAwareModelClient, + _safe_cli_summary, + bootstrap_provider_credentials, +) + + +def test_bytez_text_mapping_is_accepted_without_usage_fabrication() -> None: + """A native Bytez text field is accepted while usage remains explicitly absent.""" + set_backend(InMemoryCredentialBackend()) + try: + account = DEFAULT_PROVIDER_ACCOUNTS[2] + bootstrap_provider_credentials( + {account.credential_name: "secret-value"}, + require_all=False, + accounts=(account,), + ) + agent = ModelAgent( + "bytez_worker", + "owner/model", + account.base_url, + credential_key=account.credential_name, + provider_name="bytez", + ) + client = ProviderAwareModelClient( + bytez_request=lambda _agent, _messages, _credential: { + "output": {"text": "text-answer"} + } + ) + assert client.chat(agent, [{"role": "user", "content": "hello"}]) == "text-answer" + assert client.take_usage() is None + finally: + set_backend(None) + + +def test_safe_bootstrap_summary_exposes_names_and_counts_only() -> None: + """The CI summary is stable, JSON-serializable, and contains no unknown input fields.""" + summary = _safe_cli_summary( + { + "registered_credentials": ["OPENAI_API_KEY"], + "missing_credentials": ["BYTEZ_API_KEY"], + "secret_value": "must-not-copy", + }, + { + "candidate_model_count": 4, + "provider_accounts": {"openai_primary": {"status": "refreshed"}}, + "provider_body": "must-not-copy", + }, + ) + assert summary == { + "registered_credentials": ["OPENAI_API_KEY"], + "missing_credentials": ["BYTEZ_API_KEY"], + "candidate_model_count": 4, + "provider_accounts": {"openai_primary": {"status": "refreshed"}}, + "measurement_status": "provider_catalog_bootstrap", + } + assert "must-not-copy" not in json.dumps(summary)