diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index e4058a86c4..b213003071 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -5563,12 +5563,12 @@ jobs: } emit_known_missing_string_finding \ - "STRIX_MODEL: contextual-orchestrator/orchestrator/free" \ - "Strix PR scans must use contextual-orchestrator/orchestrator/free" \ + "STRIX_MODEL: contextual-orchestrator/orchestrator/auto" \ + "Strix PR scans must use contextual-orchestrator/orchestrator/auto" \ ".github/workflows/strix.yml" \ "scripts/ci/strix_required_workflow_smoke.sh" emit_known_missing_string_finding \ - "Strix model overrides are limited to contextual-orchestrator/orchestrator/free." \ + "Strix model overrides are limited to contextual-orchestrator/orchestrator/auto." \ "Strix model overrides must remain gateway-only" \ ".github/workflows/strix.yml" \ "scripts/ci/strix_required_workflow_smoke.sh" diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index d4bba9e50c..a0d95e5af5 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -549,14 +549,14 @@ jobs: - name: Gate Strix secrets id: gate env: - STRIX_MODEL: contextual-orchestrator/orchestrator/free + STRIX_MODEL: contextual-orchestrator/orchestrator/auto STRIX_MODEL_REQUESTED: ${{ github.event.client_payload.strix_llm || '' }} run: | requested_model="$(printf '%s' "$STRIX_MODEL_REQUESTED" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" case "$requested_model" in - ""|orchestrator/free|contextual-orchestrator/orchestrator/free) ;; + ""|orchestrator/auto|contextual-orchestrator/orchestrator/auto) ;; *) - echo '::error::Strix model overrides are limited to contextual-orchestrator/orchestrator/free.' + echo '::error::Strix model overrides are limited to contextual-orchestrator/orchestrator/auto.' exit 1 ;; esac @@ -574,6 +574,7 @@ jobs: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: ${{ steps.target_visibility.outputs.is_private }} + CONTEXTUAL_ORCHESTRATOR_POOL: auto run: | set -euo pipefail bash "$TRUSTED_STRIX_SOURCE/scripts/ci/contextual_orchestrator_review_sidecar.sh" @@ -712,11 +713,11 @@ jobs: strix_llm_file="$RUNNER_TEMP/strix_llm.txt" strix_model="$(printf '%s' "$STRIX_MODEL" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" case "$strix_model" in - orchestrator/free | contextual-orchestrator/orchestrator/free) - printf '%s' 'orchestrator/free' > "$strix_llm_file" + orchestrator/auto | contextual-orchestrator/orchestrator/auto) + printf '%s' 'orchestrator/auto' > "$strix_llm_file" ;; *) - echo '::error::STRIX_LLM must select contextual-orchestrator/orchestrator/free.' + echo '::error::STRIX_LLM must select contextual-orchestrator/orchestrator/auto.' exit 1 ;; esac @@ -739,7 +740,7 @@ jobs: LLM_API_KEY_FILE: ${{ env.LLM_API_KEY_FILE }} STRIX_TARGET_PATH: ${{ (github.event_name == 'pull_request_target' || github.event.client_payload.pr_number != '') && '__PR_SCOPE__' || './' }} STRIX_SOURCE_DIRS: ". backend frontend" - # The gateway free pool is capability-diverse. Strix function tools + # The gateway auto pool is provider-diverse. Strix function tools # must not send a provider-specific reasoning setting to every route. STRIX_REASONING_EFFORT: none STRIX_LLM_MAX_RETRIES: 1 diff --git a/AGENTS.md b/AGENTS.md index a77c90b8de..86d97ce786 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,7 +22,9 @@ provider secrets (`BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, as bootstrap transport in the same process that discovers models and serves; OpenCode and Noema use the fail-closed zero-cost pool `orchestrator/free`. Authoritative Strix analysis uses the correctness-first `orchestrator/auto` -pool. Private targets require ZDR-compliant routes in either pool, enforced by +provider-diverse pool. Non-free Strix routes +are admitted only with complete published prompt/completion price and currency +evidence, and private targets still require ZDR-compliant routes under [`scripts/ci/zdr_policy.py`](scripts/ci/zdr_policy.py). See [`docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`](docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md). diff --git a/CHANGELOG.md b/CHANGELOG.md index c6471cbf93..73af4f7ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Route required Strix scans through the contextual-orchestrator + `orchestrator/auto` pool so the five configured provider credentials form + real cross-provider failover. Priced routes require finite, nonnegative + published prompt/completion prices and an explicit currency; unknown pricing + fails closed. Private-target ZDR enforcement and the no-external-fallback + contract remain unchanged. - Allow the protected Strix required-workflow smoke to recognize only the existing `orchestrator/free` route or the provider-diverse `orchestrator/auto` route. This provides a fail-closed two-phase migration diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 092caa1d2f..4cc210a5e1 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -33,7 +33,10 @@ all five, and auto-optimize routing by cost. transport only; request-time credential reads go through the KV. 2. **Auto model discovery + governed virtual pools**: discovery runs with the orchestrator's own `discover_all_models()` against the KV credentials. - OpenCode and Noema admit only zero-priced routes. The gateway's + OpenCode and Noema admit only zero-priced routes. Strix admits provider- + diverse priced routes only when discovery supplies finite, nonnegative + prompt and completion prices plus an explicit currency; missing or malformed + price evidence fails closed. The gateway's `orchestrator/free` virtual id fails closed (`400 invalid_model`) unless an enabled zero-cost agent exists. Strix uses `orchestrator/auto`; its catalog may admit priced routes only through the same evidence-bearing policy, never @@ -48,8 +51,8 @@ all five, and auto-optimize routing by cost. fetched when egress allows it and is authoritative for the `openrouter` scope; otherwise the dated static attestation table is used, never a fabricated policy. - `scripts/ci/contextual_orchestrator_review_policy.py` turns the free-tier - discovery report into a ZDR-prioritized, provider-family-diverse agents + `scripts/ci/contextual_orchestrator_review_policy.py` turns the price- + evidenced discovery report into a ZDR-prioritized, provider-family-diverse agents catalog (primary/secondary NVIDIA keys share one outage-domain family), capped in size, in the orchestrator's own `ModelAgent` schema. 4. **Wiring**: `pr-review-autofix.yml` and the Required OpenCode dispatch @@ -59,9 +62,13 @@ all five, and auto-optimize routing by cost. `opencode.jsonc` default `model`/`small_model` is the same gateway route. `noema-review.yml` retains `orchestrator/free`. `strix.yml` provisions the same sidecar and uses the loopback chat-completions/API-compatible URL with - virtual model `orchestrator/auto`, delegating discovery, provider diversity, - and provider cost evidence to the gateway rather than embedding a second - routing policy in GitHub Actions. Strix has no external fallback and private targets pass + `orchestrator/auto`: the 2026-08-29 exact-head DiskSage scan proved that four + discovered free routes all shared the OpenRouter outage domain, which the + gateway correctly collapsed to one provider attempt. Strix therefore uses + the provider-diverse pool supplied by all five configured credentials. + Provider diversity and provider cost evidence remain delegated to the + gateway rather than embedding a second routing policy in GitHub Actions. + Strix has no external fallback and private targets pass visibility through to the gateway's ZDR requirement. Noema reviewer identity remains `NOEMA_REVIEW_TOKEN` / GitHub App / OIDC and is still never `github.token`; Autofix mutation still requires `PR_REVIEW_MERGE_TOKEN` / @@ -83,10 +90,11 @@ all five, and auto-optimize routing by cost. - The autofix/OpenCode review paths no longer hard-code any provider base URL or model id; upstream model selection is delegated to the orchestrator's - discovery + cost routing, under the zero-cost pool, with ZDR routes first. + discovery under the zero-cost pool. Strix uses the separately governed auto + pool without treating absent price metadata as paid-route evidence. - Strix delegates selection to `orchestrator/auto`. Its correctness-first pool - is distinct from the zero-cost OpenCode/Noema pool, while private-target ZDR - admission remains fail-closed. + remains distinct from the zero-cost OpenCode/Noema pool, while private-target + ZDR admission remains fail-closed. - Workers need egress to the five provider model-list hosts and, when reachable, `https://openrouter.ai/api/v1/endpoints/zdr`; the feed failure path is graceful (static table). diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index 5bc27426fe..f7bd07b343 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -63,8 +63,19 @@ def _has_text_output(model: object) -> bool: return not modalities or "text" in {str(modality).casefold() for modality in modalities} -def _free_report_rows(discovered: list[object]) -> list[dict[str, object]]: - """Convert in-process discovered models into free-only report rows. +def _route_identity(model: object) -> tuple[str, str]: + """Return the provider/model identity used to bind price evidence.""" + + return ( + str(getattr(model, "provider_name", None) or ""), + str(getattr(model, "model_id", None) or ""), + ) + + +def _report_rows( + discovered: list[object], free_route_identities: frozenset[tuple[str, str]] +) -> list[dict[str, object]]: + """Convert in-process discovered models into price-evidenced report rows. Only routes the orchestrator itself marks zero-priced (whole-prompt and whole-completion published price equal to zero; never name-implied) are @@ -73,10 +84,11 @@ def _free_report_rows(discovered: list[object]) -> list[dict[str, object]]: org ZDR policy table (``scripts/ci/zdr_policy.py``). Args: - discovered: ``discover_all_models()`` result (the free subset). + discovered: Selected ``discover_all_models()`` result. + free_route_identities: Routes the orchestrator attested as zero-priced. Returns: - Free-only rows shaped for + Price-evidenced rows shaped for ``contextual_orchestrator_review_policy.parse_discovery_report``. """ from scripts.ci import zdr_policy @@ -99,7 +111,10 @@ def _free_report_rows(discovered: list[object]) -> list[dict[str, object]]: "provider": provider, "model": model_id, "agent_id": str(getattr(model, "agent_id", None) or f"{provider}_{model_id}"), - "is_free": True, + "is_free": (provider, model_id) in free_route_identities, + "prompt_price_per_1k": getattr(model, "prompt_price_per_1k", None), + "completion_price_per_1k": getattr(model, "completion_price_per_1k", None), + "currency_code": getattr(model, "currency_code", None), "base_url": base_url, "credential_key": credential_key, "auth_scheme": auth_scheme, @@ -241,6 +256,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--preflight-out", required=True, help="Path to write sanitized runtime preflight JSON") parser.add_argument("--zdr-endpoints", default=None, help="Optional OpenRouter /api/v1/endpoints/zdr JSON path") parser.add_argument("--require-zdr", action="store_true") + parser.add_argument("--pool", choices=("free", "auto"), default="free") args = parser.parse_args(argv) from contextual_orchestrator.credentials import get_credential @@ -272,16 +288,22 @@ def main(argv: list[str] | None = None) -> int: discovered, _ = discover_all_models() except Exception as exc: # pragma: no cover - provider/networking failure is runtime-only raise SystemExit(f"review sidecar discovery failed: {exc}") from exc - free_models = [] - for model in free_discovered_models(discovered) if discovered else []: + free_models = list(free_discovered_models(discovered)) if discovered else [] + free_route_identities = frozenset(_route_identity(model) for model in free_models) + selected_models = [] + for model in discovered or []: model_id = getattr(model, "model_id", "") if not is_general_chat_agent_model_id(model_id) or not _has_text_output(model): continue - free_models.append(model) - if not free_models: - raise SystemExit("review sidecar discovered no zero-cost models; orchestrator/free would fail closed") + if args.pool == "free" and _route_identity(model) not in free_route_identities: + continue + selected_models.append(model) + if not selected_models: + raise SystemExit( + f"review sidecar discovered no eligible models; orchestrator/{args.pool} would fail closed" + ) - rows = _free_report_rows(free_models) + rows = _report_rows(selected_models, free_route_identities) _write_json(args.discovery_out, {"models": rows}) zdr_endpoints = _load_zdr_endpoints(args.zdr_endpoints) result = build_zdr_prioritized_catalog( @@ -290,6 +312,7 @@ def main(argv: list[str] | None = None) -> int: family_cap=int(os.environ.get("ORCHESTRATOR_CATALOG_FAMILY_CAP", "4")), zdr_endpoints=zdr_endpoints, require_zdr=args.require_zdr, + pool=args.pool, ) Path(args.catalog_out).write_text( json.dumps({"agents": result["agents"]}, indent=2, sort_keys=True) + "\n", diff --git a/scripts/ci/contextual_orchestrator_review_policy.py b/scripts/ci/contextual_orchestrator_review_policy.py index 66c6f305b0..01bea6cc46 100644 --- a/scripts/ci/contextual_orchestrator_review_policy.py +++ b/scripts/ci/contextual_orchestrator_review_policy.py @@ -22,6 +22,7 @@ import argparse import json +import math import os import sys import re @@ -100,6 +101,16 @@ class PolicyError(ValueError): """Raised when a discovery report cannot produce a usable catalog.""" +def _validated_price(value: object, *, route: str, field: str) -> float: + """Return a finite nonnegative published price or reject the route.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise PolicyError(f"model {route} lacks numeric {field} evidence") + price = float(value) + if not math.isfinite(price) or price < 0: + raise PolicyError(f"model {route} has invalid {field} evidence") + return price + + def parse_discovery_report(report: Mapping[str, Any]) -> list[dict[str, Any]]: """Validate and extract the model rows from a ``discover-models`` report. @@ -134,12 +145,30 @@ def parse_discovery_report(report: Mapping[str, Any]) -> list[dict[str, Any]]: candidate_id = row.get("agent_id") or f"{provider}_{model}" if not _is_valid_is_free(row.get("is_free")): raise PolicyError(f"model {provider}/{model} lacks an explicit is_free marker") + is_free = is_free_route(row.get("is_free")) + route = f"{provider}/{model}" + prompt_price = row.get("prompt_price_per_1k") + completion_price = row.get("completion_price_per_1k") + currency_code = row.get("currency_code") + if not is_free: + prompt_price = _validated_price( + prompt_price, route=route, field="prompt_price_per_1k" + ) + completion_price = _validated_price( + completion_price, route=route, field="completion_price_per_1k" + ) + if not isinstance(currency_code, str) or not currency_code.strip(): + raise PolicyError(f"model {route} lacks currency_code evidence") + currency_code = currency_code.strip().upper() normalized.append( { "provider": provider, "model": model, "agent_id": str(candidate_id), - "is_free": is_free_route(row.get("is_free")), + "is_free": is_free, + "prompt_price_per_1k": prompt_price, + "completion_price_per_1k": completion_price, + "currency_code": currency_code, "base_url": row.get("base_url") or PROVIDER_BASE_URLS[provider], "credential_key": row.get("credential_key") or PROVIDER_CREDENTIAL_NAMES[provider], "auth_scheme": row.get("auth_scheme") or PROVIDER_AUTH_SCHEMES[provider], @@ -155,8 +184,9 @@ def build_zdr_prioritized_catalog( family_cap: int = DEFAULT_FAMILY_CAP, zdr_endpoints: frozenset[str] = frozenset(), require_zdr: bool = False, + pool: str = "free", ) -> dict[str, Any]: - """Select and rank free routes into a ZDR-first, family-diverse catalog. + """Select and rank governed routes into a ZDR-first, family-diverse catalog. Ranking is deterministic and evidence-based, never heuristic cost guesses: free (zero-cost, attested by discovery price metadata) routes always outrank @@ -192,26 +222,33 @@ def family_is_open(family: str) -> bool: """Return whether a provider family still has catalog capacity.""" return per_family[family] < family_cap - all_free_rows = [row for row in rows if row["is_free"]] - free_rows = [ + if pool not in {"free", "auto"}: + raise PolicyError(f"unsupported review pool {pool!r}") + all_rows = list(rows) + all_free_rows = [row for row in all_rows if row["is_free"]] + candidate_rows = all_free_rows if pool == "free" else all_rows + eligible_rows = [ row - for row in all_free_rows + for row in candidate_rows if not require_zdr or is_zdr_model( row["provider"], model=row["model"], zdr_endpoints=zdr_endpoints ) ] - free_rows.sort( + eligible_rows.sort( key=lambda row: ( 0 if is_zdr_model( row["provider"], model=row["model"], zdr_endpoints=zdr_endpoints ) else 1, + 0 if row["is_free"] else 1, + row["provider"], + row["model"], ) ) picked: list[dict[str, Any]] = [] - for _order, row in enumerate(free_rows): + for _order, row in enumerate(eligible_rows): family = provider_family(row["provider"]) if not family_is_open(family): continue @@ -221,10 +258,10 @@ def family_is_open(family: str) -> bool: break if not picked: - route_kind = "attested ZDR free" if require_zdr else "free (zero-cost)" + route_kind = "attested ZDR" if require_zdr else pool raise PolicyError( f"no {route_kind} model route is available with the ZDR policy; " - "orchestrator/free would fail closed" + f"orchestrator/{pool} would fail closed" ) for rank, row in enumerate(picked): @@ -240,7 +277,11 @@ def family_is_open(family: str) -> bool: "base_url": row["base_url"], "api_key_env": "", "credential_key": row["credential_key"], - "tags": ["review", "cost:free", "zdr" if zdr else "non-zdr"], + "tags": [ + "review", + "cost:free" if row["is_free"] else "cost:priced", + "zdr" if zdr else "non-zdr", + ], "priority": -rank, "disabled": False, "provider_name": row["provider"], @@ -256,11 +297,13 @@ def family_is_open(family: str) -> bool: return { "agents": catalog_rows, "report": { - "pool": "orchestrator/free", + "pool": f"orchestrator/{pool}", + "total_routes": len(all_rows), "total_free_routes": len(all_free_rows), "zdr_required": require_zdr, "selected_count": len(catalog_rows), - "free_selected_count": len(picked), + "free_selected_count": sum(1 for row in picked if row["is_free"]), + "priced_selected_count": sum(1 for row in picked if not row["is_free"]), "zdr_selected_count": zdr_count, "zdr_sources": sorted( {provider_zdr_scope(row["provider"]).source for row in picked if is_zdr_model(row["provider"], model=row["model"], zdr_endpoints=zdr_endpoints)} @@ -312,6 +355,7 @@ def build_catalog_from_paths( family_cap: int = DEFAULT_FAMILY_CAP, zdr_endpoints_path: str | None = None, require_zdr: bool = False, + pool: str = "free", ) -> dict[str, Any]: """Build and persist the ZDR-prioritized ``orchestrator/free`` catalog. @@ -336,6 +380,7 @@ def build_catalog_from_paths( family_cap=family_cap, zdr_endpoints=zdr_endpoints, require_zdr=require_zdr, + pool=pool, ) Path(out_path).write_text( json.dumps({"agents": result["agents"]}, indent=2) + "\n", encoding="utf-8" @@ -356,6 +401,7 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument("--family-cap", type=int, default=DEFAULT_FAMILY_CAP) parser.add_argument("--zdr-endpoints", default=None, help="Optional OpenRouter /api/v1/endpoints/zdr JSON path") parser.add_argument("--require-zdr", action="store_true", help="Fail closed unless every selected route has attested ZDR evidence") + parser.add_argument("--pool", choices=("free", "auto"), default="free") return parser @@ -378,6 +424,7 @@ def main(argv: list[str] | None = None) -> int: family_cap=args.family_cap, zdr_endpoints_path=args.zdr_endpoints, require_zdr=args.require_zdr, + pool=args.pool, ) except (PolicyError, OSError, json.JSONDecodeError) as exc: print(f"contextual-orchestrator review policy: {exc}", file=sys.stderr) diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index b716bd55c7..bca9d5c00e 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -257,6 +257,16 @@ case "${CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR:-false}" in ;; esac +orchestrator_pool="${CONTEXTUAL_ORCHESTRATOR_POOL:-free}" +case "$orchestrator_pool" in + free|auto) + pool_args=(--pool "$orchestrator_pool") + ;; + *) + fail "CONTEXTUAL_ORCHESTRATOR_POOL must be free or auto" + ;; +esac + log "starting review sidecar on ${ORCHESTRATOR_HOST}:${ORCHESTRATOR_PORT}" cp "$ORCHESTRATOR_LAUNCHER" "$ORCHESTRATOR_WORK/launch_sidecar.py" export ORCHESTRATOR_CATALOG_LIMIT="$CATALOG_LIMIT" @@ -272,6 +282,7 @@ PYTHONPATH="$ORCHESTRATOR_SOURCE:$ORG_REPO_ROOT" \ --preflight-out "$preflight_report" \ "${zdr_args[@]}" \ "${privacy_args[@]}" \ + "${pool_args[@]}" \ > >("$sidecar_python" -u "$SIDECAR_LOG_SANITIZER" > "$sidecar_stdout") \ 2> >("$sidecar_python" -u "$SIDECAR_LOG_SANITIZER" > "$sidecar_stderr") & sidecar_pid=$! @@ -309,7 +320,9 @@ log "healthz and provider-route preflight confirmed after ${i}s (pid $sidecar_pi # process can be healthy while the coordinator/model-group path still raises an # internal error, which is the failure this contract prevents from reaching the # scanner step. -printf '%s\n' '{"model":"orchestrator/free","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Reply with just '\''OK'\''."}],"temperature":1.0,"max_tokens":16,"stream":false}' > "$gateway_preflight_request" +gateway_virtual_model="orchestrator/${orchestrator_pool}" +printf '{"model":"%s","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Reply with just '\''OK'\''."}],"temperature":1.0,"max_tokens":16,"stream":false}\n' \ + "$gateway_virtual_model" > "$gateway_preflight_request" if ! gateway_http_status="$( curl -sS --max-time 30 \ -o "$gateway_preflight_response" \ diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 6eb8322229..1243911d26 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -180,11 +180,9 @@ assert_file_contains "$full_gate_test" "assert_strix_workflow_pr_trigger_hardene assert_file_contains "$workflow_file" "Provision contextual-orchestrator Strix sidecar" "Strix workflow provisions the trusted contextual-orchestrator gateway" assert_file_contains "$workflow_file" "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" "Strix workflow binds target visibility to the gateway ZDR policy" -assert_file_contains_exactly_one_of \ - "$workflow_file" \ - "STRIX_MODEL: contextual-orchestrator/orchestrator/free" \ - "STRIX_MODEL: contextual-orchestrator/orchestrator/auto" \ - "Strix defaults every scan to an allowlisted contextual-orchestrator pool" +active_strix_models="$(sed -n -E 's/^[[:space:]]*STRIX_MODEL:[[:space:]]*([^#[:space:]]+)[[:space:]]*$/\1/p' "$workflow_file")" +[ "$active_strix_models" = "contextual-orchestrator/orchestrator/auto" ] || record_failure "Strix must define exactly one active provider-diverse auto default model" +assert_file_not_contains "$workflow_file" "STRIX_MODEL: contextual-orchestrator/orchestrator/free" "Strix must not retain the free default route" assert_file_contains "$decision_record" "authoritative Strix security analysis uses the provider-diverse \`orchestrator/auto\` pool" "The binding ADR authorizes the Strix auto route" assert_file_contains "$decision_record" "Zero Data Retention (ZDR)-compliant routes remain mandatory for private targets" "The binding ADR preserves private-target privacy" assert_file_contains "$decision_record" "Strix is intentionally correctness-first rather than zero-cost" "The binding ADR records the Strix cost boundary" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index b528e8bafc..c44e82c5ab 100644 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -313,8 +313,8 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$REPO_ROOT/tests/test_strix_repository_visibility_contract.py" "test_dispatch_api_visibility_preserves_internal_privacy" "strix visibility contract executes public, private, and internal dispatch fixtures" assert_file_contains "$workflow_file" 'STRIX_MODEL: ${{ steps.gate.outputs.strix_model }}' "strix workflow propagates the gate-selected fallback model to the scanner" assert_file_not_contains "$workflow_file" "secrets.STRIX_LLM ||" "strix workflow must not let the legacy STRIX_LLM secret override PR defaults" - assert_file_contains "$workflow_file" "Strix model overrides are limited to contextual-orchestrator/orchestrator/free" "strix workflow rejects non-gateway model overrides" - assert_file_contains "$workflow_file" "STRIX_LLM must select contextual-orchestrator/orchestrator/free" "strix workflow accepts only the gateway model" + assert_file_contains "$workflow_file" "Strix model overrides are limited to contextual-orchestrator/orchestrator/auto" "strix workflow rejects non-gateway model overrides" + assert_file_contains "$workflow_file" "STRIX_LLM must select contextual-orchestrator/orchestrator/auto" "strix workflow accepts only the gateway model" assert_file_contains "$workflow_file" 'STRIX_FALLBACK_MODELS: ""' "strix workflow disables external fallback models" assert_file_contains "$workflow_file" 'STRIX_FAIL_ON_PROVIDER_SIGNAL: "1"' "strix workflow fails closed on timeout, fatal, warning, denied, or provider failure signals" assert_file_contains "$workflow_file" 'NPM_CONFIG_IGNORE_SCRIPTS: "true"' "strix workflow disables npm lifecycle scripts for untrusted PR scan data" diff --git a/tests/test_contextual_orchestrator_review_policy.py b/tests/test_contextual_orchestrator_review_policy.py index 75a33cd4b6..12f4990daa 100644 --- a/tests/test_contextual_orchestrator_review_policy.py +++ b/tests/test_contextual_orchestrator_review_policy.py @@ -50,6 +50,9 @@ def _report() -> dict[str, object]: "model": "gpt-4.1", "agent_id": "openai_gpt_41", "is_free": False, + "prompt_price_per_1k": 0.002, + "completion_price_per_1k": 0.008, + "currency_code": "USD", }, ] } @@ -159,8 +162,9 @@ def test_parse_discovery_report_rejects_invalid_rows(report: dict[str, object]) def test_build_catalog_is_zdr_first_and_free_only() -> None: """ZDR-compliant routes outrank non-ZDR free routes; priced routes stay out.""" + parsed = policy.parse_discovery_report(_report()) result = policy.build_zdr_prioritized_catalog( - policy.parse_discovery_report(_report()), + parsed, limit=12, family_cap=4, zdr_endpoints=ZDR_FEED, @@ -180,6 +184,83 @@ def test_build_catalog_is_zdr_first_and_free_only() -> None: assert agent["credential_key"] +def test_build_auto_catalog_admits_price_evidenced_routes() -> None: + """The Strix auto pool can use priced routes without weakening the free pool.""" + parsed = policy.parse_discovery_report(_report()) + result = policy.build_zdr_prioritized_catalog( + parsed, + limit=12, + family_cap=4, + zdr_endpoints=ZDR_FEED, + pool="auto", + ) + + agents = result["agents"] + priced = next(agent for agent in agents if agent["model"] == "gpt-4.1") + assert "cost:priced" in priced["tags"] + priced_evidence = next(row for row in parsed if row["model"] == "gpt-4.1") + assert priced_evidence["prompt_price_per_1k"] == 0.002 + assert priced_evidence["completion_price_per_1k"] == 0.008 + assert priced_evidence["currency_code"] == "USD" + assert result["report"]["pool"] == "orchestrator/auto" + assert result["report"]["total_routes"] == 6 + assert result["report"]["free_selected_count"] == 5 + assert result["report"]["priced_selected_count"] == 1 + + +def test_build_auto_catalog_order_is_independent_of_discovery_order() -> None: + """Equivalent route tiers have deterministic provider/model priority.""" + parsed = policy.parse_discovery_report(_report()) + forward = policy.build_zdr_prioritized_catalog(parsed, pool="auto") + reversed_result = policy.build_zdr_prioritized_catalog(reversed(parsed), pool="auto") + assert forward["report"]["selected"] == reversed_result["report"]["selected"] + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("prompt_price_per_1k", None, "lacks numeric prompt_price_per_1k"), + ("completion_price_per_1k", -1, "invalid completion_price_per_1k"), + ("prompt_price_per_1k", float("inf"), "invalid prompt_price_per_1k"), + ("currency_code", "", "lacks currency_code"), + ], +) +def test_priced_routes_require_complete_published_price_evidence( + field: str, value: object, message: str +) -> None: + """Auto routing rejects routes whose published cost evidence is incomplete.""" + report = _report() + priced = report["models"][-1] + priced[field] = value + with pytest.raises(policy.PolicyError, match=message): + policy.parse_discovery_report(report) + + +def test_build_auto_catalog_keeps_private_targets_zdr_only() -> None: + """Private Strix auto routing still excludes every unattested route.""" + result = policy.build_zdr_prioritized_catalog( + policy.parse_discovery_report(_report()), + limit=12, + family_cap=4, + zdr_endpoints=ZDR_FEED, + require_zdr=True, + pool="auto", + ) + + assert [agent["model"] for agent in result["agents"]] == [ + "deepseek/deepseek-r1:free" + ] + assert result["report"]["priced_selected_count"] == 0 + + +def test_build_catalog_rejects_unknown_pool() -> None: + """An unrecognized virtual pool cannot silently widen model admission.""" + with pytest.raises(policy.PolicyError, match="unsupported review pool"): + policy.build_zdr_prioritized_catalog( + policy.parse_discovery_report(_report()), pool="direct" + ) + + def test_build_catalog_assigns_unique_priorities() -> None: """Each selected agent gets a distinct priority so TaskOrchestrator cannot tie on id.""" result = policy.build_zdr_prioritized_catalog( @@ -245,7 +326,15 @@ def test_build_catalog_fails_closed_without_free_models() -> None: """An empty free pool cannot serve orchestrator/free and must fail loudly.""" report = { "models": [ - {"provider": "openai", "model": "gpt-4.1", "agent_id": "oa_41", "is_free": False} + { + "provider": "openai", + "model": "gpt-4.1", + "agent_id": "oa_41", + "is_free": False, + "prompt_price_per_1k": 0.002, + "completion_price_per_1k": 0.008, + "currency_code": "USD", + } ] } with pytest.raises(policy.PolicyError, match="no free"): diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index 99eb4aff4b..a61caeee0e 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -150,7 +150,11 @@ def test_sidecar_preserves_diagnostics_and_probes_the_real_gateway() -> None: assert 'gateway_preflight_response="$ORCHESTRATOR_WORK/gateway-preflight.json"' in sidecar assert '"http://${ORCHESTRATOR_HOST}:${ORCHESTRATOR_PORT}/v1/chat/completions"' in sidecar assert 'Authorization: Bearer ${ORCHESTRATOR_TOKEN}' in sidecar - assert '"model":"orchestrator/free"' in sidecar + assert 'orchestrator_pool="${CONTEXTUAL_ORCHESTRATOR_POOL:-free}"' in sidecar + assert 'gateway_virtual_model="orchestrator/${orchestrator_pool}"' in sidecar + assert '"model":"%s"' in sidecar + assert '"$gateway_virtual_model" > "$gateway_preflight_request"' in sidecar + assert '"model":"orchestrator/free"' not in sidecar assert "gateway preflight returned unusable chat content" in sidecar assert 'SIDECAR_LOG_SANITIZER="$ORG_REPO_ROOT/scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py"' in sidecar assert '"$sidecar_python" -u "$SIDECAR_LOG_SANITIZER" > "$sidecar_stdout"' in sidecar diff --git a/tests/test_contextual_orchestrator_review_sidecar_contract.py b/tests/test_contextual_orchestrator_review_sidecar_contract.py index 499c12b466..a1c0746f03 100644 --- a/tests/test_contextual_orchestrator_review_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_review_sidecar_contract.py @@ -259,7 +259,7 @@ def test_sidecar_masks_gateway_token_before_startup_can_emit_logs() -> None: mask_index = text.index(mask) for later_operation in ( "git clone", - "python3 -m pip install", + '"$sidecar_python" -m pip install', '"$ORCHESTRATOR_WORK/launch_sidecar.py"', "healthz", ): @@ -276,8 +276,8 @@ def test_launcher_registers_secrets_into_the_kv_once() -> None: assert "get_credential(REVIEW_AUTH_CREDENTIAL_NAME)" in text -def test_launcher_uses_orchestrator_discovery_and_free_pool() -> None: - """Discovery, free filtering, and serving come from the vendored library.""" +def test_launcher_uses_orchestrator_discovery_and_governed_pools() -> None: + """Discovery, price evidence, and serving come from the vendored library.""" text = _read(LAUNCHER) assert "from contextual_orchestrator.chat_capability import is_general_chat_agent_model_id" in text assert "from contextual_orchestrator.model_discovery import discover_all_models, free_discovered_models" in text @@ -294,9 +294,29 @@ def test_launcher_uses_orchestrator_discovery_and_free_pool() -> None: assert has_text_output(SimpleNamespace(output_modalities=("text", "image"))) assert not has_text_output(SimpleNamespace(output_modalities=("video",))) assert not has_text_output(SimpleNamespace()) + report_rows = launcher["_report_rows"] + free = SimpleNamespace( + provider_name="openrouter", + model_id="free/model", + agent_id="openrouter_free_model", + output_modalities=("text",), + ) + priced = SimpleNamespace( + provider_name="openai", + model_id="priced-model", + agent_id="openai_priced_model", + output_modalities=("text",), + prompt_price_per_1k=0.002, + completion_price_per_1k=0.008, + currency_code="USD", + ) + rows = report_rows([free, priced], frozenset({("openrouter", "free/model")})) + assert [row["is_free"] for row in rows] == [True, False] + assert rows[1]["prompt_price_per_1k"] == 0.002 assert "from contextual_orchestrator.orchestrator import ModelClient, TaskOrchestrator, load_agents" in text assert "from contextual_orchestrator.server import SecurityConfig, serve" in text - assert "orchestrator/free would fail closed" in text + assert 'parser.add_argument("--pool", choices=("free", "auto"), default="free")' in text + assert "orchestrator/{args.pool} would fail closed" in text assert "scripts.ci.contextual_orchestrator_review_policy" in text assert "from scripts.ci import zdr_policy" in text @@ -326,6 +346,7 @@ def test_strix_gateway_uses_provider_neutral_reasoning_effort() -> None: """Gateway free-pool scans must not force unsupported provider controls.""" text = _read(STRIX_WORKFLOW) assert "STRIX_REASONING_EFFORT: none" in text + assert "CONTEXTUAL_ORCHESTRATOR_POOL: auto" in text def test_sidecar_probes_the_pinned_server_body_limit_at_http_boundary() -> None: @@ -431,7 +452,7 @@ def test_required_strix_uses_the_gateway_and_zdr_visibility_contract() -> None: workflow = _read(STRIX_WORKFLOW) assert "Provision contextual-orchestrator Strix sidecar" in workflow assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in workflow - assert 'STRIX_MODEL: contextual-orchestrator/orchestrator/free' in workflow + assert 'STRIX_MODEL: contextual-orchestrator/orchestrator/auto' in workflow assert "provider_mode=contextual_orchestrator" in workflow assert "STRIX_LLM_DEFAULT_PROVIDER: contextual_orchestrator" in workflow assert workflow.index("Resolve target repository visibility") < workflow.index( diff --git a/tests/test_noema_orchestrator_workflow_contract.py b/tests/test_noema_orchestrator_workflow_contract.py index 481b3356aa..dfa9aa2c8f 100644 --- a/tests/test_noema_orchestrator_workflow_contract.py +++ b/tests/test_noema_orchestrator_workflow_contract.py @@ -87,7 +87,7 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed(tmp_path: Path) -> env={ **os.environ, "GITHUB_OUTPUT": str(strix_output), - "STRIX_MODEL": "contextual-orchestrator/orchestrator/free", + "STRIX_MODEL": "contextual-orchestrator/orchestrator/auto", "STRIX_MODEL_REQUESTED": "", }, capture_output=True, @@ -96,12 +96,12 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed(tmp_path: Path) -> ) assert strix.returncode == 0, strix.stderr assert { - "strix_model=contextual-orchestrator/orchestrator/free", + "strix_model=contextual-orchestrator/orchestrator/auto", "enabled=true", "provider_mode=contextual_orchestrator", } <= set(strix_output.read_text().splitlines()) assert ( - "STRIX_MODEL: contextual-orchestrator/orchestrator/free" + "STRIX_MODEL: contextual-orchestrator/orchestrator/auto" in workflow_text("strix.yml") ) assert ( diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 2aba930965..de764f440a 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -19,7 +19,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "e4058a86c48527df458e6b4d3e5e3d696b738b1c" +REVIEW_DISPATCH_BLOB_SHA = "b2130030715ef5661f62d5c098d3cf3f49d1686d" def _workflow_text(path: Path) -> str: diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 3992a3b20b..9a7794d4eb 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -576,7 +576,7 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed( env={ **os.environ, "GITHUB_OUTPUT": str(strix_output), - "STRIX_MODEL": "contextual-orchestrator/orchestrator/free", + "STRIX_MODEL": "contextual-orchestrator/orchestrator/auto", "STRIX_MODEL_REQUESTED": "", }, capture_output=True, @@ -585,12 +585,12 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed( ) assert strix.returncode == 0, strix.stderr assert { - "strix_model=contextual-orchestrator/orchestrator/free", + "strix_model=contextual-orchestrator/orchestrator/auto", "enabled=true", "provider_mode=contextual_orchestrator", } <= set(strix_output.read_text().splitlines()) assert ( - "STRIX_MODEL: contextual-orchestrator/orchestrator/free" + "STRIX_MODEL: contextual-orchestrator/orchestrator/auto" in workflow_text("strix.yml") ) assert ( diff --git a/tests/test_strix_contextual_orchestrator_contract.py b/tests/test_strix_contextual_orchestrator_contract.py index 185f8c1cc9..0db9f6b4ef 100644 --- a/tests/test_strix_contextual_orchestrator_contract.py +++ b/tests/test_strix_contextual_orchestrator_contract.py @@ -26,7 +26,7 @@ def setUp(self) -> None: def test_default_scan_provisions_the_existing_gateway_sidecar(self) -> None: """Every scan uses the five-provider gateway, never a direct pool.""" self.assertIn("Provision contextual-orchestrator Strix sidecar", self.workflow) - self.assertIn("STRIX_MODEL: contextual-orchestrator/orchestrator/free", self.workflow) + self.assertIn("STRIX_MODEL: contextual-orchestrator/orchestrator/auto", self.workflow) self.assertIn("provider_mode=contextual_orchestrator", self.workflow) self.assertIn("STRIX_FALLBACK_MODELS: \"\"", self.workflow) self.assertNotIn( @@ -48,7 +48,7 @@ def test_model_override_cannot_escape_the_gateway(self) -> None: """A dispatch payload cannot select a direct provider route.""" self.assertIn("github.event.client_payload.strix_llm", self.workflow) self.assertIn( - "Strix model overrides are limited to contextual-orchestrator/orchestrator/free", + "Strix model overrides are limited to contextual-orchestrator/orchestrator/auto", self.workflow, ) for direct_route in ("nvidia_nim/*)", "openrouter/free", "openai-direct/gpt-5.4"): @@ -73,7 +73,11 @@ def test_gateway_install_is_hash_locked_and_token_is_masked(self) -> None: def test_required_smoke_pins_the_gateway_default(self) -> None: """The bounded required-path smoke rejects a future direct-default regression.""" self.assertIn("contextual-orchestrator Strix sidecar", self.smoke) - self.assertIn("STRIX_MODEL: contextual-orchestrator/orchestrator/free", self.smoke) + self.assertIn("active_strix_models=", self.smoke) + self.assertIn( + '"$active_strix_models" = "contextual-orchestrator/orchestrator/auto"', + self.smoke, + ) self.assertIn("Strix does not resolve a direct provider outside the gateway", self.smoke) def test_required_smoke_rejects_invalid_sidecar_syntax(self) -> None: diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index ba8344455b..07ab21bad4 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -190,7 +190,7 @@ def test_workflow_routes_all_scans_through_contextual_orchestrator(self) -> None workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") self.assertIn("Provision contextual-orchestrator Strix sidecar", workflow) - self.assertIn("STRIX_MODEL: contextual-orchestrator/orchestrator/free", workflow) + self.assertIn("STRIX_MODEL: contextual-orchestrator/orchestrator/auto", workflow) self.assertIn("provider_mode=contextual_orchestrator", workflow) self.assertIn("STRIX_LLM_DEFAULT_PROVIDER: contextual_orchestrator", workflow) self.assertNotIn("Resolve live NVIDIA NIM Strix models", workflow) @@ -201,7 +201,7 @@ def test_workflow_rejects_non_gateway_model_overrides(self) -> None: workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") self.assertIn("STRIX_MODEL_REQUESTED", workflow) - self.assertIn("Strix model overrides are limited to contextual-orchestrator/orchestrator/free.", workflow) + self.assertIn("Strix model overrides are limited to contextual-orchestrator/orchestrator/auto.", workflow) self.assertIn("STRIX_FALLBACK_MODELS: \"\"", workflow) def test_outer_workflow_requires_litellm_context_for_nvidia_404(self) -> None: