diff --git a/README.md b/README.md index 5679581f3..bb62b2a47 100644 --- a/README.md +++ b/README.md @@ -318,6 +318,8 @@ python tests/test_chat_transport_role_separation.py python tests/test_chat_capability_unknown_identifiers.py python tests/test_chat_passthrough_capability_isolation.py python tests/test_discovery_bootstrap_selection.py +python tests/test_chat_capability.py +python tests/test_review_gateway.py python tests/test_provider_bootstrap.py python tests/test_provider_bootstrap_secret_normalization.py python tests/test_provider_catalog_bootstrap.py diff --git a/contextual_orchestrator/review_gateway.py b/contextual_orchestrator/review_gateway.py new file mode 100644 index 000000000..ef2e6438c --- /dev/null +++ b/contextual_orchestrator/review_gateway.py @@ -0,0 +1,148 @@ +"""Bootstrap a local authenticated gateway for trusted CI model reviews. + +The five provider keys arrive from a deployment environment only as bootstrap +transport. They are immediately registered in the process-local KV, after +which model discovery and every provider request use the normal credential +registry path. This keeps the review sidecar useful without pretending that a +short-lived runner has a durable production credential store. +""" + +from __future__ import annotations + +import argparse +import os +from dataclasses import replace +from typing import Mapping + +from .credentials import NotConfigured, get_credential, register_credential +from .cost_ledger import PriceBook +from .chat_capability import is_general_chat_agent_model_id +from .kv_config import InMemoryConfigStore +from .model_discovery import ( + agent_from_discovered, + discover_all_models, + refresh_price_book, + select_bootstrap_discovered_agents, +) +from .orchestrator import ModelClient, TaskOrchestrator +from .server import SecurityConfig, serve + +REVIEW_CREDENTIAL_NAMES: tuple[str, ...] = ( + "BYTEZ_API_KEY", + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", +) +DEFAULT_REVIEW_AGENT_LIMIT = 12 +REVIEW_AUTH_CREDENTIAL_NAME = "CONTEXTUAL_ORCHESTRATOR_TOKEN" + + +def register_review_credentials(environment: Mapping[str, str]) -> tuple[str, ...]: + """Register non-empty CI provider and gateway credentials in the KV. + + ``environment`` is bootstrap input only. The returned names contain no + secret values and are suitable for diagnostics or tests. + """ + registered: list[str] = [] + for name in REVIEW_CREDENTIAL_NAMES: + value = environment.get(name, "").strip() + if value: + register_credential(name, value) + registered.append(name) + auth_value = environment.get(REVIEW_AUTH_CREDENTIAL_NAME, "").strip() + if auth_value: + register_credential(REVIEW_AUTH_CREDENTIAL_NAME, auth_value) + registered.append(REVIEW_AUTH_CREDENTIAL_NAME) + return tuple(registered) + + +def build_review_orchestrator( + environment: Mapping[str, str] | None = None, + *, + max_agents: int = DEFAULT_REVIEW_AGENT_LIMIT, +) -> TaskOrchestrator: + """Build an enabled, cost-ranked orchestrator from discovered providers. + + A missing provider key is allowed so the sidecar can use an available + subset. Starting with no provider key or no discovered model fails closed + rather than silently falling back to a mock or fabricated model. + """ + if type(max_agents) is not int or max_agents < 1: + raise ValueError("max_agents must be a positive integer") + source_environment = os.environ if environment is None else environment + registered = register_review_credentials(source_environment) + if not any(name in registered for name in REVIEW_CREDENTIAL_NAMES): + raise NotConfigured("review gateway requires at least one provider credential") + + discovered, errors = discover_all_models() + if not discovered: + providers = ", ".join(sorted({error.provider_name for error in errors})) + detail = f"; failed providers: {providers}" if providers else "" + raise NotConfigured(f"review gateway discovered no provider models{detail}") + + chat_discovered = [model for model in discovered if is_general_chat_agent_model_id(model.model_id)] + if not chat_discovered: + raise NotConfigured("review gateway discovered no general chat models") + price_book = PriceBook(InMemoryConfigStore()) + refresh_price_book(chat_discovered, price_book) + selected = select_bootstrap_discovered_agents(chat_discovered, price_book, max_agents) + if not selected: + raise NotConfigured("review gateway selected no provider models") + + agents = [ + replace( + agent_from_discovered(model, priority=index), + disabled=False, + # The discovery catalog does not advertise reasoning, coding, or + # verification support. ``review`` is the gateway purpose, not a + # fabricated provider capability. + tags=("review",), + # TaskOrchestrator ranks larger priorities first. Discovery is + # already cheapest-first, so preserve that order for routing. + priority=-index, + ) + for index, model in enumerate(selected) + ] + return TaskOrchestrator( + agents, + client=ModelClient(max_output_tokens=32768), + ) + + +def _build_parser() -> argparse.ArgumentParser: + """Create the sidecar CLI parser with loopback-safe defaults.""" + parser = argparse.ArgumentParser(description="Run the contextual review gateway sidecar.") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=18080) + parser.add_argument("--max-agents", type=int, default=DEFAULT_REVIEW_AGENT_LIMIT) + parser.add_argument( + "--auth-token", + default="", + help="Explicit local bearer token; otherwise resolve --auth-token-key from the KV.", + ) + parser.add_argument("--auth-token-key", default=REVIEW_AUTH_CREDENTIAL_NAME) + return parser + + +def main() -> None: + """Discover providers and serve the authenticated OpenAI-compatible sidecar.""" + parser = _build_parser() + args = parser.parse_args() + if args.max_agents < 1: + parser.error("--max-agents must be a positive integer") + register_review_credentials(os.environ) + auth_token = args.auth_token or get_credential(args.auth_token_key) + if not auth_token: + raise SystemExit(f"KV credential {args.auth_token_key!r} or --auth-token is required") + orchestrator = build_review_orchestrator(max_agents=args.max_agents) + serve( + orchestrator, + host=args.host, + port=args.port, + security=SecurityConfig(auth_token=auth_token), + ) + + +if __name__ == "__main__": # pragma: no cover + main() diff --git a/docs/doctoring/request-framing-boundary.md b/docs/doctoring/request-framing-boundary.md index 2d21d77e0..ed4737172 100644 --- a/docs/doctoring/request-framing-boundary.md +++ b/docs/doctoring/request-framing-boundary.md @@ -30,4 +30,3 @@ negative-length raw sockets, chunked raw sockets, and a short declared body. Internet Engineering Task Force. (2022). *HTTP/1.1* (RFC 9112). RFC Editor. https://www.rfc-editor.org/rfc/rfc9112.html - diff --git a/docs/kv-credentials.md b/docs/kv-credentials.md index d63113c1f..1b40fbafe 100644 --- a/docs/kv-credentials.md +++ b/docs/kv-credentials.md @@ -234,12 +234,10 @@ uses (`TaskOrchestrator.sync_discovered_agents`, an idempotent upsert of added **disabled**, so a newly found model never starts serving traffic before an operator opts it in. -Cost-based auto-optimization reuses the existing price-table selector rather -than adding a new one: `model_discovery.refresh_price_book` writes any -provider-reported per-token pricing into `PriceBook`, and -`model_discovery.select_cheapest_discovered_agent` calls -`batch_routing.cheapest_upstream` to pick the lowest-cost candidate among -discovered models for a representative request shape. +Cost-based auto-optimization uses `model_discovery.refresh_price_book` to write +provider-reported per-token pricing into `PriceBook`. Discovery selectors rank +complete, valid price evidence first and keep unknown-priced candidates as a +deterministic fallback. ## Gateway direction diff --git a/tests/test_chat_capability.py b/tests/test_chat_capability.py new file mode 100644 index 000000000..0660d5ddf --- /dev/null +++ b/tests/test_chat_capability.py @@ -0,0 +1,73 @@ +"""Boundary tests for model-family chat and orchestration-role capability gates.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.chat_capability import ( # noqa: E402 + is_chat_compatible_model_id, + is_general_chat_agent_model_id, +) + + +@pytest.mark.parametrize( + "model_id", + ( + "bge-small-en", + "clip-vit", + "dall-e-3", + "e5-large", + "embed-model", + "embedding-model", + "embeddings-v2", + "gte-base", + "image-model", + "images-v2", + "moderation-latest", + "realtime-preview", + "rerank-v3", + "reranker-large", + "siglip-base", + "sora-video", + "speech-model", + "transcribe-large", + "transcription-v2", + "tts-1", + "whisper-1", + ), +) +def test_endpoint_only_model_families_cannot_use_chat_transport(model_id: str) -> None: + """Reject every explicitly known endpoint-only model family.""" + assert is_chat_compatible_model_id(model_id) is False + assert is_general_chat_agent_model_id(model_id) is False + + +@pytest.mark.parametrize("model_id", ("embedder-v2", "moderator-v1", "reranking-v1", "transcriber-v1")) +def test_endpoint_prefixes_reject_unlisted_model_variants(model_id: str) -> None: + """Reject future-looking family variants without matching exact tokens.""" + assert is_chat_compatible_model_id(model_id) is False + + +def test_empty_and_non_string_identifiers_fail_closed() -> None: + """Unknown identifier shapes never become eligible by accident.""" + assert is_chat_compatible_model_id("") is False + assert is_chat_compatible_model_id(None) is False # type: ignore[arg-type] + assert is_general_chat_agent_model_id("") is False + + +@pytest.mark.parametrize("model_id", ("safety-classifier", "guard-model", "shieldgemma-2b", "nemo_guard_v2")) +def test_transport_compatible_policy_models_stay_out_of_general_agent_roles(model_id: str) -> None: + """Safety and guard models may use chat transport but are not thinkers.""" + assert is_chat_compatible_model_id(model_id) is True + assert is_general_chat_agent_model_id(model_id) is False + + +def test_normal_chat_identifier_remains_eligible() -> None: + """A normal provider model remains eligible for both ordinary gates.""" + assert is_chat_compatible_model_id("provider/gpt-5.5") is True + assert is_general_chat_agent_model_id("provider/gpt-5.5") is True diff --git a/tests/test_discovery_bootstrap_selection.py b/tests/test_discovery_bootstrap_selection.py index 45ca53917..a6124fee3 100644 --- a/tests/test_discovery_bootstrap_selection.py +++ b/tests/test_discovery_bootstrap_selection.py @@ -120,6 +120,18 @@ def test_persisted_price_row_missing_one_component_remains_unknown() -> None: assert price_book.get_price("partial_vendor", "partial-model") is None +def test_persisted_boolean_price_row_remains_unknown() -> None: + """Do not coerce corrupt KV booleans into zero-cost price evidence.""" + store = InMemoryConfigStore() + store.set( + "llm_price_entries", + "broken_vendor:broken-model", + {"prompt_price_per_1k": False, "completion_price_per_1k": False}, + ) + + assert PriceBook(store).get_price("broken_vendor", "broken-model") is None + + def test_invalid_catalog_prices_are_unknown_not_trusted_cost_evidence() -> None: """Reject negative, non-finite, and boolean provider price values.""" assert model_discovery._price_per_1k("-0.000001") is None diff --git a/tests/test_review_gateway.py b/tests/test_review_gateway.py new file mode 100644 index 000000000..995fb8f9e --- /dev/null +++ b/tests/test_review_gateway.py @@ -0,0 +1,227 @@ +"""Tests for the trusted CI review gateway bootstrap.""" + +from __future__ import annotations + +import sys + +import pytest + +from contextual_orchestrator.credentials import ( + InMemoryCredentialBackend, + NotConfigured, + get_credential, + set_backend, +) +from contextual_orchestrator.chat_capability import ( + is_chat_compatible_model_id, + is_general_chat_agent_model_id, +) +from contextual_orchestrator.model_discovery import DiscoveredModel +from contextual_orchestrator import review_gateway + + +@pytest.fixture(autouse=True) +def _fresh_backend(): + """Give every gateway test an isolated in-memory credential registry.""" + set_backend(InMemoryCredentialBackend()) + try: + yield + finally: + set_backend(None) + + +def _discovered(provider: str, model: str, credential: str, price: float) -> DiscoveredModel: + """Build one deterministic discovered chat candidate for the tests.""" + return DiscoveredModel( + provider_name=provider, + model_id=model, + credential_name=credential, + chat_base_url=f"https://{provider}.example/v1", + auth_scheme="Bearer", + prompt_price_per_1k=price, + completion_price_per_1k=price, + ) + + +def test_build_review_orchestrator_registers_all_provider_credentials(monkeypatch): + """The bootstrap registers every configured provider key before discovery.""" + discovered = [ + _discovered("openai", "gpt-review", "OPENAI_API_KEY", 0.01), + _discovered("openrouter", "router-review", "OPENROUTER_API_KEY", 0.02), + _discovered("nvidia_nim", "nim-review", "NVIDIA_NIM_API_KEY", 0.03), + ] + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: (discovered, [])) + environment = { + name: f"secret-{name.lower()}" + for name in review_gateway.REVIEW_CREDENTIAL_NAMES + } + + orchestrator = review_gateway.build_review_orchestrator(environment, max_agents=2) + + assert len(orchestrator.agents) == 2 + assert all(not agent.disabled for agent in orchestrator.agents) + assert all(agent.tags == ("review",) for agent in orchestrator.agents) + assert all(get_credential(name) == environment[name] for name in review_gateway.REVIEW_CREDENTIAL_NAMES) + + +def test_build_review_orchestrator_routes_to_cheapest_selected_agent(monkeypatch): + """Cost-ranked discovery remains the routing order after agent construction.""" + discovered = [ + _discovered("openai", "expensive_review", "OPENAI_API_KEY", 2.0), + _discovered("openrouter", "cheap_review", "OPENROUTER_API_KEY", 0.01), + ] + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: (discovered, [])) + + orchestrator = review_gateway.build_review_orchestrator( + {"OPENAI_API_KEY": "openai-secret", "OPENROUTER_API_KEY": "router-secret"}, + max_agents=2, + ) + + assert orchestrator._select_agent("review this change", "worker").model == "cheap_review" + + +def test_build_review_orchestrator_keeps_provider_diverse_failover(monkeypatch): + """The gateway uses independent provider families before duplicate providers.""" + discovered = [ + _discovered("openrouter", "cheap_first", "OPENROUTER_API_KEY", 0.01), + _discovered("openrouter", "cheap_second", "OPENROUTER_API_KEY", 0.02), + _discovered("openai", "independent_review", "OPENAI_API_KEY", 1.0), + ] + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: (discovered, [])) + + orchestrator = review_gateway.build_review_orchestrator( + {"OPENAI_API_KEY": "openai-secret", "OPENROUTER_API_KEY": "router-secret"}, + max_agents=2, + ) + + assert [agent.model for agent in orchestrator.agents] == [ + "cheap_first", + "independent_review", + ] + + +def test_build_review_orchestrator_excludes_endpoint_only_models(monkeypatch): + """Embedding and image catalog rows never enter the review-agent pool.""" + discovered = [ + _discovered("openai", "text-embedding-3-large", "OPENAI_API_KEY", 0.001), + _discovered("openai", "gpt-review", "OPENAI_API_KEY", 2.0), + ] + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: (discovered, [])) + + orchestrator = review_gateway.build_review_orchestrator( + {"OPENAI_API_KEY": "openai-secret"}, max_agents=1 + ) + + assert [agent.model for agent in orchestrator.agents] == ["gpt-review"] + + +def test_build_review_orchestrator_fails_closed_without_general_chat_models(monkeypatch): + """A catalog containing only endpoint-specific models cannot start review.""" + discovered = [_discovered("openai", "text-embedding-3-large", "OPENAI_API_KEY", 0.001)] + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: (discovered, [])) + + with pytest.raises(NotConfigured, match="general chat models"): + review_gateway.build_review_orchestrator({"OPENAI_API_KEY": "openai-secret"}) + + +@pytest.mark.parametrize( + ("model_id", "chat_compatible", "general_agent"), + [ + ("text-embedding-3-large", False, False), + ("gpt-image-1", False, False), + ("gpt-review", True, True), + ("nvidia/llama-3.1-nemoguard-8b-content-safety", True, False), + (None, False, False), + ("", False, False), + ], +) +def test_chat_capability_boundary_is_explicit( + model_id, chat_compatible: bool, general_agent: bool +) -> None: + """The gateway rejects endpoint-only and specialized role identifiers.""" + assert is_chat_compatible_model_id(model_id) is chat_compatible + assert is_general_chat_agent_model_id(model_id) is general_agent + + +def test_build_review_orchestrator_fails_closed_without_credentials(): + """A sidecar cannot start without at least one registered provider key.""" + with pytest.raises(NotConfigured, match="provider credential"): + review_gateway.build_review_orchestrator({}) + + +def test_build_review_orchestrator_does_not_count_auth_as_provider_credential(): + """The gateway auth token cannot satisfy the provider bootstrap gate.""" + with pytest.raises(NotConfigured, match="provider credential"): + review_gateway.build_review_orchestrator( + {review_gateway.REVIEW_AUTH_CREDENTIAL_NAME: "local-review-token"} + ) + + +def test_build_review_orchestrator_rejects_invalid_agent_limit(): + """The sidecar refuses an invalid routing-pool bound before using credentials.""" + with pytest.raises(ValueError, match="max_agents"): + review_gateway.build_review_orchestrator({}, max_agents=0) + + +def test_build_review_orchestrator_fails_closed_without_discovered_models(monkeypatch): + """A configured key without a usable model list cannot start the sidecar.""" + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: ([], [])) + with pytest.raises(NotConfigured, match="no provider models"): + review_gateway.build_review_orchestrator({"OPENAI_API_KEY": "openai-secret"}) + + +def test_build_review_orchestrator_fails_closed_when_selection_is_empty(monkeypatch): + """A discovery result must still produce at least one selected candidate.""" + monkeypatch.setattr( + review_gateway, + "discover_all_models", + lambda: ([_discovered("openai", "gpt-review", "OPENAI_API_KEY", 0.01)], []), + ) + monkeypatch.setattr(review_gateway, "select_bootstrap_discovered_agents", lambda *args: []) + + with pytest.raises(NotConfigured, match="selected no provider models"): + review_gateway.build_review_orchestrator({"OPENAI_API_KEY": "openai-secret"}) + + +def test_main_starts_authenticated_gateway(monkeypatch): + """The CLI passes the generated local token into the protected server.""" + discovered = [_discovered("openai", "gpt-review", "OPENAI_API_KEY", 0.01)] + monkeypatch.setattr(review_gateway, "discover_all_models", lambda: (discovered, [])) + captured: dict[str, object] = {} + + def fake_serve(orchestrator, **kwargs): + """Capture server startup without binding a test port.""" + captured["orchestrator"] = orchestrator + captured.update(kwargs) + + monkeypatch.setattr(review_gateway, "serve", fake_serve) + monkeypatch.setenv("OPENAI_API_KEY", "openai-secret") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "local-review-token") + monkeypatch.setattr(sys, "argv", ["review_gateway", "--port", "18181"]) + + review_gateway.main() + + assert captured["host"] == "127.0.0.1" + assert captured["port"] == 18181 + security = captured["security"] + assert security.auth_token == "local-review-token" + assert security.allow_public_bind is False + assert get_credential(review_gateway.REVIEW_AUTH_CREDENTIAL_NAME) == "local-review-token" + + +def test_main_requires_authentication(monkeypatch): + """The sidecar never starts an unauthenticated inference endpoint.""" + monkeypatch.delenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", raising=False) + monkeypatch.setattr(sys, "argv", ["review_gateway"]) + + with pytest.raises(SystemExit, match="CONTEXTUAL_ORCHESTRATOR_TOKEN"): + review_gateway.main() + + +def test_main_rejects_invalid_agent_limit_without_traceback(monkeypatch, capsys): + """Argparse reports an invalid routing-pool bound as a CLI usage error.""" + monkeypatch.setattr(sys, "argv", ["review_gateway", "--max-agents", "0"]) + with pytest.raises(SystemExit) as exc_info: + review_gateway.main() + assert exc_info.value.code == 2 + assert "--max-agents must be a positive integer" in capsys.readouterr().err