fix(discovery): bootstrap an honest provider-diverse failover pool

.github/workflows/provider-catalog-sync.yml

@@ -0,0 +1,105 @@
+name: Provider catalog sync
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "17 * * * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: provider-catalog-sync
+  cancel-in-progress: false
+
+jobs:
+  sync:
+    name: Bootstrap durable provider KV and model catalog
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    environment: production
+    timeout-minutes: 15
+    steps:
+      - name: Checkout protected default branch
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install hash-pinned runtime dependencies
+        run: python -m pip install --disable-pip-version-check --no-input --require-hashes -r requirements.lock
+
+      - name: Register credentials and refresh normalized model catalog
+        shell: bash
+        env:
+          CONTEXTUAL_ORCHESTRATOR_KV_BACKEND: postgres
+          CONTEXTUAL_ORCHESTRATOR_KV_DSN: ${{ secrets.CONTEXTUAL_ORCHESTRATOR_KV_DSN }}
+          CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE: ${{ secrets.CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE }}
+          NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}
+          NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }}
+          BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          set -euo pipefail
+          test -n "${CONTEXTUAL_ORCHESTRATOR_KV_DSN}"
+          test -n "${CONTEXTUAL_ORCHESTRATOR_KV_PASSPHRASE}"
+          python -m contextual_orchestrator.provider_catalog_bootstrap --model-limit 24 > provider-bootstrap-report.json
+          python - <<'PY'
+          import json
+          from pathlib import Path
+
+          report = json.loads(Path('provider-bootstrap-report.json').read_text(encoding='utf-8'))
+          expected = {
+              'NVIDIA_NIM_API_KEY',
+              'NVIDIA_NIM_API_KEY_SUB',
+              'BYTEZ_API_KEY',
+              'OPENROUTER_API_KEY',
+              'OPENAI_API_KEY',
+          }
+          registered = set(report['registered_credentials'])
+          if registered != expected:
+              raise SystemExit(f'credential inventory mismatch: {sorted(expected - registered)}')
+          if report['catalog_backend'] != 'postgres':
+              raise SystemExit('provider catalog is not durable PostgreSQL')
+          if report['catalog_model_count'] < 1 or report['eligible_model_count'] < 1:
+              raise SystemExit('provider catalog has no compatible serving model')
+          if not report['selected_agent_ids']:
+              raise SystemExit('provider catalog produced no serving candidates')
+          if report['enabled_agent_ids'] or report['durable_agent_pool']:
+              raise SystemExit('ephemeral Actions sync must not claim agent-pool activation')
+          print(json.dumps({
+              'registered_credentials': sorted(registered),
+              'live_discovered_model_count': report['live_discovered_model_count'],
+              'catalog_model_count': report['catalog_model_count'],
+              'last_known_good_model_count': report['last_known_good_model_count'],
+              'selected_agent_count': len(report['selected_agent_ids']),
+              'catalog_refresh_failure_count': report['catalog_refresh_failure_count'],
+              'providers_with_errors': report['providers_with_errors'],
+          }, sort_keys=True))
+          PY
+          python - <<'PY'
+          import os
+          from pathlib import Path
+
+          report = Path('provider-bootstrap-report.json').read_text(encoding='utf-8')
+          names = (
+              'NVIDIA_NIM_API_KEY',
+              'NVIDIA_NIM_API_KEY_SUB',
+              'BYTEZ_API_KEY',
+              'OPENROUTER_API_KEY',
+              'OPENAI_API_KEY',
+          )
+          leaked = [
+              name
+              for name in names
+              if os.environ[name].rstrip('\r\n')
+              and os.environ[name].rstrip('\r\n') in report
+          ]
+          if leaked:
+              raise SystemExit(f'provider bootstrap report leaked secret values for: {leaked}')
+          PY

contextual_orchestrator/__main__.py

@@ -16,7 +16,7 @@
     agent_id_for,
     discover_all_models,
     refresh_price_book,
-    select_top_n_cheapest_discovered_agents,
+    select_bootstrap_discovered_agents,
 )
 from .orchestrator import (
     CONTEXTUAL_ORCHESTRATOR_CONTRACT_V1,
@@ -211,7 +211,7 @@ def _discover_models_command(argv: list[str]) -> None:
         type=_non_negative_int,
         default=0,
         metavar="N",
-        help="Enable the N cheapest discovered agents in --agents-db (auto-optimization bootstrap; "
+        help="Enable a price-honest, provider-diverse discovered agent pool in --agents-db (auto-optimization bootstrap; "
         "requires --agents-db; 0 disables, the default, leaving every discovered agent inert).",
     )
     args = parser.parse_args(argv)
@@ -229,7 +229,7 @@ def _discover_models_command(argv: list[str]) -> None:
         )
         bootstrap.sync_discovered_agents([agent_from_discovered(model) for model in discovered])
         if args.enable_cheapest:
-            for model in select_top_n_cheapest_discovered_agents(discovered, price_book, args.enable_cheapest):
+            for model in select_bootstrap_discovered_agents(discovered, price_book, args.enable_cheapest):
                 agent_id = agent_id_for(model)
                 bootstrap.patch_agent("default", agent_id, {"status": "active"})
                 enabled_agent_ids.append(agent_id)

contextual_orchestrator/cost_ledger.py

@@ -31,6 +31,7 @@
 import threading
 from dataclasses import dataclass, field
 from decimal import ROUND_HALF_UP, Decimal
+import math
 import time
 from typing import Any, Dict, List, Optional, Protocol
 import uuid
@@ -162,11 +163,30 @@ def get_price(self, provider: str, model: str) -> Optional[PriceEntry]:
             raw = self._config.get(_PRICE_CATEGORY, _price_key(provider, "*"), None)
         if raw is None:
             return None
+        if not isinstance(raw, dict):
+            return None
+        try:
+            if (
+                "prompt_price_per_1k" not in raw
+                or "completion_price_per_1k" not in raw
+            ):
+                return None
+            prompt_price = float(raw["prompt_price_per_1k"])
+            completion_price = float(raw["completion_price_per_1k"])
+        except (OverflowError, TypeError, ValueError):
+            return None
+        if (
+            not math.isfinite(prompt_price)
+            or not math.isfinite(completion_price)
+            or prompt_price < 0
+            or completion_price < 0
+        ):
+            return None
         return PriceEntry(
             provider_name=raw.get("provider_name", provider),
             model_name=raw.get("model_name", model),
-            prompt_price_per_1k=float(raw.get("prompt_price_per_1k", 0.0)),
-            completion_price_per_1k=float(raw.get("completion_price_per_1k", 0.0)),
+            prompt_price_per_1k=prompt_price,
+            completion_price_per_1k=completion_price,
             currency_code=raw.get("currency_code", self.default_currency),
         )
 

contextual_orchestrator/credentials.py

@@ -50,6 +50,10 @@ def set(self, name: str, value: str) -> None:
         """Register (or replace) the secret stored under ``name``."""
         ...
 
+    def delete(self, name: str) -> None:
+        """Remove one credential after an unvalidated candidate promotion."""
+        ...
+
 
 class InMemoryCredentialBackend:
     """Process-local credential registry for dev and tests (no Postgres needed)."""
@@ -68,6 +72,11 @@ def set(self, name: str, value: str) -> None:
         with self._lock:
             self._store[name] = value
 
+    def delete(self, name: str) -> None:
+        """Remove ``name`` from the in-memory credential registry if present."""
+        with self._lock:
+            self._store.pop(name, None)
+
 
 # --- Postgres pgcrypto-encrypted credential registry ------------------------
 #
@@ -112,6 +121,15 @@ def __init__(self, dsn: str, passphrase: str) -> None:
         self._passphrase = passphrase
         self._ensured = False
 
+    @property
+    def connection_dsn(self) -> str:
+        """Return the bootstrap DSN for a colocated metadata store.
+
+        Callers must treat this as connection material: never include it in logs,
+        reports, traces, or exceptions. Provider API keys remain inaccessible.
+        """
+        return self._dsn
+
     @classmethod
     def from_env(cls) -> "PostgresCredentialBackend":
         """Build the backend from bootstrap transport env vars (the only allowed env use).
@@ -173,6 +191,17 @@ def set(self, name: str, value: str) -> None:  # pragma: no cover - requires a l
                 )
             conn.commit()
 
+    def delete(self, name: str) -> None:  # pragma: no cover - requires a live Postgres
+        """Delete one encrypted credential after a failed candidate promotion."""
+        with self._connect() as conn:
+            self._ensure_schema(conn)
+            with conn.cursor() as cur:
+                cur.execute(
+                    "DELETE FROM provider_credentials WHERE credential_name = %s",
+                    (name,),
+                )
+            conn.commit()
+
 
 _backend: CredentialBackend | None = None
 _backend_lock = threading.Lock()
@@ -216,3 +245,8 @@ def get_credential(name: str) -> str | None:
 def register_credential(name: str, value: str) -> None:
     """Register a named secret into the KV (used by the bootstrap CLI)."""
     get_backend().set(name, value)
+
+
+def delete_credential(name: str) -> None:
+    """Remove a named credential from the KV after an unvalidated promotion."""
+    get_backend().delete(name)