From 6369bef089b40b630abbd54b596d5b0f1cc6fb87 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 12:40:12 +0900 Subject: [PATCH] fix(ci): keep sidecar credential accounts independent Pin the review sidecar to contextual-orchestrator model-group semantics and stop collapsing separately credentialed accounts into provider families. Signed-off-by: Seongho Bae Commit-Message-Assisted-by: Claude (via Claude Code) --- CHANGELOG.md | 5 ++ ...ntextual-orchestrator-vendored-free-zdr.md | 13 ++-- .../0005-sidecar-preflight-token-budget.md | 2 +- ...contextual_orchestrator_review_launcher.py | 22 +++--- .../contextual_orchestrator_review_policy.py | 59 +++++++-------- .../contextual_orchestrator_review_sidecar.sh | 56 ++------------ ...estrator_review_live_discovery_contract.py | 2 +- ...t_contextual_orchestrator_review_policy.py | 73 +++++++++---------- ...l_orchestrator_review_runtime_preflight.py | 14 ++-- ...al_orchestrator_review_sidecar_contract.py | 2 +- 10 files changed, 99 insertions(+), 149 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6682241627..d31f4760e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Noema, Strix, and OpenCode review sidecars now vendor contextual-orchestrator + at `0adca4703df67f8f31d3ea5b04a1e07ed775dd6c` and treat every KV credential + as an independent discovery account. Same-vendor credentials no longer + collapse into a provider family; only explicit model groups may share + routing evidence. - Web verification now runs backend, frontend, and E2E commands inside an isolated Linux bubblewrap workspace by default (`--isolation required`), mounting a read-only runtime root with a single writable `/workspace` diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 7e0ee41395..5b4f130f3f 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -24,7 +24,7 @@ all five, and auto-optimize routing by cost. 1. **Vendoring, pinned**: `scripts/ci/contextual_orchestrator_review_sidecar.sh` clones `ContextualWisdomLab/contextual-orchestrator` at an exact SHA - (`30c6d71680e659f25a0a433d4726ad0d437f9757` today) into `RUNNER_TEMP`. The + (`0adca4703df67f8f31d3ea5b04a1e07ed775dd6c` today) into `RUNNER_TEMP`. The source's `requirements.lock` is installed with `--require-hashes` and `--no-deps`, so dependency resolution cannot silently move the reviewed runtime. @@ -70,9 +70,10 @@ all five, and auto-optimize routing by cost. primary, and the admitted priced tier remains fallback-only. `scripts/ci/contextual_orchestrator_review_policy.py` turns the discovery report into a free-first, cost-evidence-ranked, ZDR-prioritized, - provider-family-diverse agents catalog (primary/secondary NVIDIA keys share - one outage-domain family), capped in size, in the orchestrator's own - `ModelAgent` schema. + credential-account-diverse agents catalog, capped in size, in the + orchestrator's own `ModelAgent` schema. Every KV credential is an independent + account; vendor or endpoint identity does not imply model equivalence. Only + explicit `model_group` membership may share routing evidence. 4. **Wiring**: `pr-review-autofix.yml` and the Required OpenCode dispatch provision the sidecar with the five secrets before OpenCode runs and point every model/diagnosis candidate at `contextual-orchestrator/orchestrator/free`; @@ -177,8 +178,8 @@ all five, and auto-optimize routing by cost. other caller that opts into it explicitly — this amendment only removes it as Strix's default and as an accepted Strix override value. - **Monitoring evidence for the accepted risk above:** `scripts/ci/contextual_orchestrator_review_policy.py` - now reports `free_family_diversity` in the catalog report — the count of - distinct outage-domain provider families (see `provider_family`) among + now reports `free_account_diversity` in the catalog report — the count of + independently credentialed accounts (see `provider_account`) among *all* discovered free routes, independent of which pool is requested. This was drafted (in a now-superseded addendum proposing to gate the `free` decision on this evidence rather than making it directly) before the diff --git a/docs/adr/0005-sidecar-preflight-token-budget.md b/docs/adr/0005-sidecar-preflight-token-budget.md index 933f44e31f..f024bc9933 100644 --- a/docs/adr/0005-sidecar-preflight-token-budget.md +++ b/docs/adr/0005-sidecar-preflight-token-budget.md @@ -69,7 +69,7 @@ exists: > "모델마다 max_tokens 허용치가 다 다른데" — each model has a genuinely different max_tokens allowance. `orchestrator/free` is a heterogeneous pool (`nvidia_nim`, `openai`, `opencode_zen`, `bytez`, -`openrouter`, ... — see `contextual_orchestrator_review_policy.py`'s `PROVIDER_FAMILIES`), and which +`openrouter`, ... — see `contextual_orchestrator_review_policy.py`'s credential table), and which candidate a given preflight run draws varies. A fixed `max_tokens` is wrong on two independent, evidenced axes for a pool like this: diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index 7a6ac17d9f..6dbbe2d5e3 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -12,7 +12,7 @@ in-process (so the KV-backed credentials are visible to it), the zero-cost ("free") routes are collected into a report, and ``scripts/ci/contextual_orchestrator_review_policy.py`` turns that report into a -ZDR-prioritized, provider-family-diverse catalog for ``orchestrator/free``. +ZDR-prioritized, credential-account-diverse catalog for ``orchestrator/free``. Keeping the decision logic in that stdlib-only module lets every branch of the ZDR policy be tested offline in this repository while ``orchestrator/free`` still resolves from authentically zero-priced models discovered by the @@ -669,17 +669,17 @@ def _with_discovery_counts( report: dict[str, object], rows: list[dict[str, Any]], *, - provider_family: Any, + provider_account: Any, ) -> dict[str, object]: """Copy a stage report while restoring full discovery-tier counts. - ``free_family_diversity`` is recomputed here from the full discovery-wide + ``free_account_diversity`` is recomputed here from the full discovery-wide ``rows``, not trusted from the stage report: the primary ``auto``-pool stage may have selected only ZDR-admitted free rows (undercounting diversity whenever ``--require-zdr`` excludes some free routes) and the priced-fallback stage selects only priced rows (so its own internally computed diversity is always zero) -- either stage report's - ``free_family_diversity``, as returned by ``build_zdr_prioritized_catalog`` + ``free_account_diversity``, as returned by ``build_zdr_prioritized_catalog`` from whatever narrower row set it was given, would otherwise contradict that field's documented "among *all* discovered free routes" contract. """ @@ -690,9 +690,9 @@ def _with_discovery_counts( "total_free_routes": sum(row.get("cost_evidence") == "free" for row in rows), "total_priced_routes": sum(row.get("cost_evidence") == "priced" for row in rows), "total_unknown_routes": sum(row.get("cost_evidence") == "unknown" for row in rows), - "free_family_diversity": len( + "free_account_diversity": len( { - provider_family(str(row["provider"])) + provider_account(str(row["provider"])) for row in rows if row.get("cost_evidence") == "free" } @@ -780,7 +780,7 @@ def main(argv: list[str] | None = None) -> int: build_zdr_prioritized_catalog, is_zdr_model, parse_discovery_report, - provider_family, + provider_account, ) registered = register_review_credentials(os.environ) @@ -848,13 +848,13 @@ def main(argv: list[str] | None = None) -> int: result = build_zdr_prioritized_catalog( primary_rows, limit=primary_limit, - family_cap=int(os.environ.get("ORCHESTRATOR_CATALOG_FAMILY_CAP", "4")), + account_cap=int(os.environ.get("ORCHESTRATOR_CATALOG_ACCOUNT_CAP", "4")), zdr_endpoints=zdr_endpoints, require_zdr=args.require_zdr, pool=args.pool, ) result["report"] = _with_discovery_counts( - result["report"], normalized_rows, provider_family=provider_family + result["report"], normalized_rows, provider_account=provider_account ) Path(args.catalog_out).write_text( json.dumps({"agents": result["agents"]}, indent=2, sort_keys=True) + "\n", @@ -879,7 +879,7 @@ def main(argv: list[str] | None = None) -> int: fallback_result = build_zdr_prioritized_catalog( admitted_priced_rows, limit=fallback_limit, - family_cap=int(os.environ.get("ORCHESTRATOR_CATALOG_FAMILY_CAP", "4")), + account_cap=int(os.environ.get("ORCHESTRATOR_CATALOG_ACCOUNT_CAP", "4")), zdr_endpoints=zdr_endpoints, require_zdr=args.require_zdr, pool="auto", @@ -888,7 +888,7 @@ def main(argv: list[str] | None = None) -> int: fallback_result = None if fallback_result is not None: fallback_result["report"] = _with_discovery_counts( - fallback_result["report"], normalized_rows, provider_family=provider_family + fallback_result["report"], normalized_rows, provider_account=provider_account ) fallback_result["report"]["primary_selected_count"] = primary_report[ "selected_count" diff --git a/scripts/ci/contextual_orchestrator_review_policy.py b/scripts/ci/contextual_orchestrator_review_policy.py index 6a03319f00..1c8a170144 100644 --- a/scripts/ci/contextual_orchestrator_review_policy.py +++ b/scripts/ci/contextual_orchestrator_review_policy.py @@ -27,13 +27,8 @@ route_key, ) -PROVIDER_FAMILIES: Mapping[str, str] = { - "nvidia_nim": "nvidia_nim", - "nvidia_nim_sub": "nvidia_nim", -} - DEFAULT_CATALOG_LIMIT = 12 -DEFAULT_FAMILY_CAP = 4 +DEFAULT_ACCOUNT_CAP = 4 COST_FREE = "free" COST_PRICED = "priced" @@ -51,9 +46,9 @@ class PolicyError(ValueError): """Raised when discovery evidence cannot produce a governed catalog.""" -def provider_family(provider_name: str) -> str: - """Return the outage-domain family for a provider.""" - return PROVIDER_FAMILIES.get(provider_name, provider_name) +def provider_account(provider_name: str) -> str: + """Return the independently credentialed provider account identity.""" + return provider_name def _normalize_agent_id(candidate: str, provider_name: str) -> str: @@ -198,27 +193,23 @@ def build_zdr_prioritized_catalog( rows: Iterable[Mapping[str, Any]], *, limit: int = DEFAULT_CATALOG_LIMIT, - family_cap: int = DEFAULT_FAMILY_CAP, + account_cap: int = DEFAULT_ACCOUNT_CAP, zdr_endpoints: frozenset[str] = frozenset(), require_zdr: bool = False, pool: str = "free", ) -> dict[str, Any]: - """Select a free-first, ZDR-aware, provider-family-diverse catalog. - - The returned report's ``free_family_diversity`` counts the distinct - outage-domain families (see ``provider_family``) among *all* discovered - free routes, independent of ``pool`` or the per-family selection cap. - A caller deciding whether a CI consumer may run on a strict, fail-closed - ``orchestrator/free`` pool without an ``orchestrator/auto`` paid-route - safety net should require at least two independent families here — one - family alone (e.g. every free route sharing a single upstream provider, - as recorded for Strix in ADR-0003) means that provider's outage takes - the whole free catalog down with it. + """Select a free-first, ZDR-aware, credential-account-diverse catalog. + + The returned report's ``free_account_diversity`` counts the distinct + credential accounts among *all* discovered free routes, independent of + ``pool`` or the per-account selection cap. Vendor identity is not model + equivalence; only an explicit contextual-orchestrator ``model_group`` may + share routing evidence across routes. This counts routes discovery reports as free, not routes runtime preflight has confirmed are actually serving requests: a value of two or - more is evidence that a family-outage cannot immediately empty the free - catalog, not proof that either family is presently reachable. A caller + more is evidence that one account failure cannot immediately empty the free + catalog, not proof that either account is presently reachable. A caller needing readiness, not just discovery-time diversity, must combine this with the runtime preflight report the sidecar already produces. """ @@ -257,13 +248,13 @@ def build_zdr_prioritized_catalog( ) ) - per_family: Counter[str] = Counter() + per_account: Counter[str] = Counter() picked: list[Mapping[str, Any]] = [] for row in eligible_rows: - family = provider_family(str(row["provider"])) - if per_family[family] >= family_cap: + account = provider_account(str(row["provider"])) + if per_account[account] >= account_cap: continue - per_family[family] += 1 + per_account[account] += 1 picked.append(row) if len(picked) >= limit: break @@ -310,8 +301,8 @@ def build_zdr_prioritized_catalog( } ) - free_family_diversity = len( - {provider_family(str(row["provider"])) for row in all_free_rows} + free_account_diversity = len( + {provider_account(str(row["provider"])) for row in all_free_rows} ) selected_evidence = [_cost_evidence(row) for row in picked] @@ -323,7 +314,7 @@ def build_zdr_prioritized_catalog( "total_free_routes": len(all_free_rows), "total_priced_routes": len(all_priced_rows), "total_unknown_routes": len(all_unknown_rows), - "free_family_diversity": free_family_diversity, + "free_account_diversity": free_account_diversity, "zdr_required": require_zdr, "selected_count": len(catalog_rows), "free_selected_count": selected_evidence.count(COST_FREE), @@ -381,7 +372,7 @@ def build_catalog_from_paths( out_path: str, report_path: str, limit: int = DEFAULT_CATALOG_LIMIT, - family_cap: int = DEFAULT_FAMILY_CAP, + account_cap: int = DEFAULT_ACCOUNT_CAP, zdr_endpoints_path: str | None = None, require_zdr: bool = False, pool: str = "free", @@ -391,7 +382,7 @@ def build_catalog_from_paths( result = build_zdr_prioritized_catalog( parse_discovery_report(report), limit=limit, - family_cap=family_cap, + account_cap=account_cap, zdr_endpoints=_load_zdr_endpoints(zdr_endpoints_path), require_zdr=require_zdr, pool=pool, @@ -418,7 +409,7 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument("--out", required=True, help="Path to write agents JSON") parser.add_argument("--report", required=True, help="Path to write audit JSON") parser.add_argument("--limit", type=int, default=DEFAULT_CATALOG_LIMIT) - parser.add_argument("--family-cap", type=int, default=DEFAULT_FAMILY_CAP) + parser.add_argument("--account-cap", type=int, default=DEFAULT_ACCOUNT_CAP) parser.add_argument("--zdr-endpoints", default=None) parser.add_argument("--require-zdr", action="store_true") parser.add_argument("--pool", choices=("free", "auto"), default="free") @@ -434,7 +425,7 @@ def main(argv: list[str] | None = None) -> int: out_path=args.out, report_path=args.report, limit=args.limit, - family_cap=args.family_cap, + account_cap=args.account_cap, zdr_endpoints_path=args.zdr_endpoints, require_zdr=args.require_zdr, pool=args.pool, diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index c82a230c1a..34cf89eb22 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -9,12 +9,12 @@ # are registered into the process-local KV by the launcher in the SAME process # that performs live model discovery and serves requests — never read back at # request time. The in-process free-priced discovery evidence is turned into a -# ZDR-prioritized, provider-family-diverse agents catalog by +# ZDR-prioritized, credential-account-diverse agents catalog by # scripts/ci/contextual_orchestrator_review_policy.py for the `orchestrator/free` # (fail-closed zero-cost) pool. set -euo pipefail -ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-30c6d71680e659f25a0a433d4726ad0d437f9757}" +ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-0adca4703df67f8f31d3ea5b04a1e07ed775dd6c}" ORCHESTRATOR_GIT_URL="${ORCHESTRATOR_GIT_URL:-https://github.com/ContextualWisdomLab/contextual-orchestrator.git}" # The Strix gate and Noema SSRF guard accept this one process-local origin. # Keep it fixed so an environment override cannot create an unvalidated sidecar. @@ -36,51 +36,11 @@ SIDECAR_LOG_SANITIZER="$ORG_REPO_ROOT/scripts/ci/sanitize_contextual_orchestrato # of guessing whether the async sanitizer has caught up. SIDECAR_DISCOVERY_DIAGNOSTICS_SENTINEL="discovery_diagnostics_complete" CATALOG_LIMIT="${ORCHESTRATOR_CATALOG_LIMIT:-12}" -# 2026-08-30: raised from 4. contextual_orchestrator_review_policy.py's -# family_cap groups nvidia_nim and nvidia_nim_sub as one outage-domain family -# and, per an exact-head evidence trail, currently that single family is the -# *only* one populating orchestrator/free (46 free rows, 100% nvidia_nim* -- -# 23 distinct model ids shared by both keys). Candidate selection sorts -# eligible rows alphabetically by (provider, model) with no reliability -# awareness, so a family_cap of 4 deterministically admitted the same four -# alphabetically-first candidates on every run -- always including two -# NVIDIA-retired model ids (google/gemma-3-12b-it, google/gemma-3-4b-it; -# confirmed HTTP 404 on live preflight) plus two others that timed out in the -# same recovered run -- while never giving the other ~19 healthy free -# nvidia_nim* models in the same run's own discovery report a chance. This is -# not throughput tuning: it is the confirmed, reproducible root cause of -# orchestrator/free's "no provider route passed the Strix plain-chat -# preflight" failures (see docs/product-technical-gap-baseline.md's -# 2026-08-30 sidecar-preflight entries for the full evidence, including the -# exact discovery/preflight artifact this comment is based on). -# 8 is a deliberately moderate raise, not a wholesale removal of the cap. The -# picking loop below also stops at CATALOG_LIMIT (12) total regardless of -# family_cap, so the absolute worst case across any number of families was -# already REVIEW_PREFLIGHT_TIMEOUT_SECONDS (10s) x 12 = 120s before this -# change (reached once family_cap x distinct-families >= 12, i.e. >=3 -# families at the old cap of 4) and stays 120s after it -- this raise does -# not move that pre-existing ceiling. What it does change is when that -# ceiling is reached and the typical case today: with the single family -# (nvidia_nim) that currently fills 100% of orchestrator/free, worst-case -# preflight time rises from ~40s (4 candidates) to ~80s (8 candidates); with -# exactly two distinct families it would now also reach the 120s ceiling -# (previously ~80s at family_cap=4). Both figures stay within the sidecar's -# existing 180s readiness-wait budget in the common case; this was reasoned -# from, not verified against, live provider timing, since this session has -# no access to the five provider credentials the sidecar's KV requires. If -# real hosted -# runs show this is still insufficient (all 8 still failing) or the added -# latency itself becomes the bottleneck, the more complete fix is a live -# provider /v1/models cross-check at discovery time to drop retired model ids -# before they ever reach preflight -- see git history's now-removed -# select_nvidia_nim_model.py (removed in #1442) for a worked example of that -# exact query-the-provider-catalog pattern, applied there to a different, -# direct-provider caller -- rather than raising this further. The PR number -# is used here, not a raw commit SHA or the removing branch's name: a squash -# merge would leave a raw pre-merge commit unreachable in plain git once the -# branch is deleted, while the PR itself (and its full commit history) stays -# permanently resolvable on GitHub. -CATALOG_FAMILY_CAP="${ORCHESTRATOR_CATALOG_FAMILY_CAP:-8}" +# Each KV credential is an independent account, including two credentials for +# the same vendor or endpoint. The account cap prevents one credential from +# consuming the bounded twelve-route preflight catalog without inventing a +# provider-family equivalence relation. +CATALOG_ACCOUNT_CAP="${ORCHESTRATOR_CATALOG_ACCOUNT_CAP:-8}" ORCHESTRATOR_GITHUB_ENV="${GITHUB_ENV:-}" sidecar_python="$(command -v python3)" @@ -320,7 +280,7 @@ esac log "starting review sidecar on ${ORCHESTRATOR_HOST}:${ORCHESTRATOR_PORT}" cp "$ORCHESTRATOR_LAUNCHER" "$ORCHESTRATOR_WORK/launch_sidecar.py" export ORCHESTRATOR_CATALOG_LIMIT="$CATALOG_LIMIT" -export ORCHESTRATOR_CATALOG_FAMILY_CAP="$CATALOG_FAMILY_CAP" +export ORCHESTRATOR_CATALOG_ACCOUNT_CAP="$CATALOG_ACCOUNT_CAP" # Stream stdout/stderr through the redacting sanitizer as two named, awaitable # processes (not bare `> >(...)` substitutions, whose PIDs bash never exposes) # so a failure handler can wait for the sanitizer to finish flushing before it diff --git a/tests/test_contextual_orchestrator_review_live_discovery_contract.py b/tests/test_contextual_orchestrator_review_live_discovery_contract.py index 32af68a30a..627af2d879 100644 --- a/tests/test_contextual_orchestrator_review_live_discovery_contract.py +++ b/tests/test_contextual_orchestrator_review_live_discovery_contract.py @@ -91,7 +91,7 @@ def test_auto_pool_excludes_unknown_cost_and_remains_provider_diverse() -> None: result = policy.build_zdr_prioritized_catalog( policy.parse_discovery_report(_live_discovery_report()), limit=6, - family_cap=2, + account_cap=2, zdr_endpoints=PRICED_ZDR_FEED, pool="auto", ) diff --git a/tests/test_contextual_orchestrator_review_policy.py b/tests/test_contextual_orchestrator_review_policy.py index 0f0f8cb6b0..b10fc4a0b9 100644 --- a/tests/test_contextual_orchestrator_review_policy.py +++ b/tests/test_contextual_orchestrator_review_policy.py @@ -68,11 +68,11 @@ def _report() -> dict[str, object]: } -def test_provider_family_groups_nvidia_keys() -> None: - """The primary and secondary NVIDIA keys share one outage-domain family.""" - assert policy.provider_family("nvidia_nim") == "nvidia_nim" - assert policy.provider_family("nvidia_nim_sub") == "nvidia_nim" - assert policy.provider_family("openai") == "openai" +def test_provider_account_keeps_nvidia_keys_independent() -> None: + """The primary and secondary NVIDIA credentials are separate accounts.""" + assert policy.provider_account("nvidia_nim") == "nvidia_nim" + assert policy.provider_account("nvidia_nim_sub") == "nvidia_nim_sub" + assert policy.provider_account("openai") == "openai" @pytest.mark.parametrize( @@ -187,7 +187,7 @@ def test_build_catalog_is_zdr_first_and_free_only() -> None: result = policy.build_zdr_prioritized_catalog( parsed, limit=12, - family_cap=4, + account_cap=4, zdr_endpoints=ZDR_FEED, ) agents = result["agents"] @@ -211,7 +211,7 @@ def test_build_auto_catalog_admits_price_evidenced_routes() -> None: result = policy.build_zdr_prioritized_catalog( parsed, limit=12, - family_cap=4, + account_cap=4, zdr_endpoints=ZDR_FEED, pool="auto", ) @@ -262,7 +262,7 @@ def test_build_auto_catalog_keeps_private_targets_zdr_only() -> None: result = policy.build_zdr_prioritized_catalog( policy.parse_discovery_report(_report()), limit=12, - family_cap=4, + account_cap=4, zdr_endpoints=ZDR_FEED, require_zdr=True, pool="auto", @@ -274,27 +274,19 @@ def test_build_auto_catalog_keeps_private_targets_zdr_only() -> None: assert result["report"]["priced_selected_count"] == 0 -def test_build_catalog_reports_free_family_diversity() -> None: - """Diversity counts distinct outage-domain families among all free routes.""" +def test_build_catalog_reports_free_account_diversity() -> None: + """Diversity counts independently credentialed accounts with free routes.""" result = policy.build_zdr_prioritized_catalog( policy.parse_discovery_report(_report()), limit=12, - family_cap=4, + account_cap=4, zdr_endpoints=ZDR_FEED, ) - # openrouter, nvidia_nim (+ its nvidia_nim_sub sibling), openai, bytez. - assert result["report"]["free_family_diversity"] == 4 + assert result["report"]["free_account_diversity"] == 5 -def test_build_catalog_reports_single_family_free_concentration() -> None: - """A free catalog sharing one outage domain reports diversity of one. - - Regression coverage for the 2026-08-29 Strix finding recorded in - ADR-0003: every discovered free route sharing one upstream provider - means that provider's outage takes down the whole free catalog, which - is why Strix cannot safely run on a strict ``orchestrator/free`` pool - without this evidence showing at least two independent families. - """ +def test_build_catalog_counts_same_vendor_credentials_independently() -> None: + """Same-vendor credentials remain distinct discovery accounts.""" single_family_report = { "models": [ { @@ -316,9 +308,9 @@ def test_build_catalog_reports_single_family_free_concentration() -> None: result = policy.build_zdr_prioritized_catalog( policy.parse_discovery_report(single_family_report), limit=12, - family_cap=4, + account_cap=4, ) - assert result["report"]["free_family_diversity"] == 1 + assert result["report"]["free_account_diversity"] == 2 def test_build_catalog_rejects_unknown_pool() -> None: @@ -334,7 +326,7 @@ def test_build_catalog_assigns_unique_priorities() -> None: result = policy.build_zdr_prioritized_catalog( policy.parse_discovery_report(_report()), limit=12, - family_cap=4, + account_cap=4, zdr_endpoints=ZDR_FEED, ) priorities = [agent["priority"] for agent in result["agents"]] @@ -344,8 +336,8 @@ def test_build_catalog_assigns_unique_priorities() -> None: assert result["report"]["total_free_routes"] == 5 -def test_build_catalog_applies_family_cap() -> None: - """A family cap keeps one outage domain from absorbing the pool.""" +def test_build_catalog_applies_account_cap() -> None: + """An account cap keeps one credential from absorbing the pool.""" report = { "models": [ {"provider": "nvidia_nim", "model": f"m{i}", "agent_id": f"nim_a{i}", "is_free": True, **FREE_PRICE} @@ -367,14 +359,15 @@ def test_build_catalog_applies_family_cap() -> None: ] } result = policy.build_zdr_prioritized_catalog( - policy.parse_discovery_report(report), limit=12, family_cap=2 + policy.parse_discovery_report(report), limit=12, account_cap=2 ) - family_counts: dict[str, int] = {} + account_counts: dict[str, int] = {} for agent in result["agents"]: - family = policy.provider_family(agent["provider_name"]) - family_counts[family] = family_counts.get(family, 0) + 1 - assert family_counts["nvidia_nim"] == 2 - assert family_counts["openai"] == 2 + account = policy.provider_account(agent["provider_name"]) + account_counts[account] = account_counts.get(account, 0) + 1 + assert account_counts["nvidia_nim"] == 2 + assert account_counts["nvidia_nim_sub"] == 2 + assert account_counts["openai"] == 2 def test_build_catalog_respects_limit() -> None: @@ -386,7 +379,7 @@ def test_build_catalog_respects_limit() -> None: ] } result = policy.build_zdr_prioritized_catalog( - policy.parse_discovery_report(report), limit=5, family_cap=100 + policy.parse_discovery_report(report), limit=5, account_cap=100 ) assert len(result["agents"]) == 5 @@ -408,14 +401,14 @@ def test_build_catalog_fails_closed_without_free_models() -> None: } with pytest.raises(policy.PolicyError, match="no free"): policy.build_zdr_prioritized_catalog( - policy.parse_discovery_report(report), limit=12, family_cap=4 + policy.parse_discovery_report(report), limit=12, account_cap=4 ) def test_build_catalog_uses_static_table_without_feed() -> None: """Without a feed, OpenRouter is not granted ZDR for every free route.""" result = policy.build_zdr_prioritized_catalog( - policy.parse_discovery_report(_report()), limit=12, family_cap=4 + policy.parse_discovery_report(_report()), limit=12, account_cap=4 ) assert result["report"]["zdr_endpoints_feed_used"] is False assert result["report"]["zdr_selected_count"] == 0 @@ -472,7 +465,7 @@ def test_build_catalog_from_paths_writes_both_files(tmp_path) -> None: out_path=str(catalog), report_path=str(report), limit=12, - family_cap=4, + account_cap=4, zdr_endpoints_path=str(feed), ) assert catalog.exists() @@ -497,7 +490,7 @@ def test_main_success_writes_catalog(tmp_path) -> None: str(report), "--limit", "12", - "--family-cap", + "--account-cap", "4", ] ) @@ -549,7 +542,7 @@ def test_private_catalog_admits_only_attested_zdr_routes() -> None: result = policy.build_zdr_prioritized_catalog( policy.parse_discovery_report(_report()), limit=12, - family_cap=4, + account_cap=4, zdr_endpoints=ZDR_FEED, require_zdr=True, ) @@ -567,6 +560,6 @@ def test_private_catalog_fails_closed_without_attested_zdr_route() -> None: policy.build_zdr_prioritized_catalog( policy.parse_discovery_report(_report()), limit=12, - family_cap=4, + account_cap=4, require_zdr=True, ) diff --git a/tests/test_contextual_orchestrator_review_runtime_preflight.py b/tests/test_contextual_orchestrator_review_runtime_preflight.py index d689cfa08d..32f1c22413 100644 --- a/tests/test_contextual_orchestrator_review_runtime_preflight.py +++ b/tests/test_contextual_orchestrator_review_runtime_preflight.py @@ -1486,13 +1486,13 @@ def test_discovery_counts_survive_stage_specific_policy_reports() -> None: {"cost_evidence": "unknown", "provider": "bytez"}, ] enriched = namespace["_with_discovery_counts"]( - base, rows, provider_family=policy.provider_family + base, rows, provider_account=policy.provider_account ) assert base == {"selected_count": 1, "selected": [{"model": "priced/model"}]} assert [enriched[key] for key in ( "total_routes", "total_free_routes", "total_priced_routes", "total_unknown_routes" )] == [4, 1, 2, 1] - assert enriched["free_family_diversity"] == 1 + assert enriched["free_account_diversity"] == 1 def test_discovery_counts_recompute_diversity_from_full_discovery_not_the_stage() -> None: @@ -1500,13 +1500,13 @@ def test_discovery_counts_recompute_diversity_from_full_discovery_not_the_stage( Regression for a real bug: the ``auto``-pool primary stage only sees ZDR-admitted free rows, and the priced-fallback stage sees no free rows - at all, so either stage's internally computed ``free_family_diversity`` + at all, so either stage's internally computed ``free_account_diversity`` (whatever ``build_zdr_prioritized_catalog`` returned from its own narrower input) would undercount or read zero even when the full - discovery has multi-family free-route diversity. + discovery has multiple credential accounts with free routes. """ namespace = _load_launcher() - stage_report_from_priced_only_rows = {"free_family_diversity": 0} + stage_report_from_priced_only_rows = {"free_account_diversity": 0} full_discovery_rows = [ {"cost_evidence": "free", "provider": "nvidia_nim"}, {"cost_evidence": "free", "provider": "openrouter"}, @@ -1515,9 +1515,9 @@ def test_discovery_counts_recompute_diversity_from_full_discovery_not_the_stage( enriched = namespace["_with_discovery_counts"]( stage_report_from_priced_only_rows, full_discovery_rows, - provider_family=policy.provider_family, + provider_account=policy.provider_account, ) - assert enriched["free_family_diversity"] == 2 + assert enriched["free_account_diversity"] == 2 def test_temporary_fallback_catalog_is_removed_after_loading(tmp_path: Path) -> None: diff --git a/tests/test_contextual_orchestrator_review_sidecar_contract.py b/tests/test_contextual_orchestrator_review_sidecar_contract.py index af7472057c..11807185e4 100644 --- a/tests/test_contextual_orchestrator_review_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_review_sidecar_contract.py @@ -40,7 +40,7 @@ ) GATEWAY_MODEL = "contextual-orchestrator/orchestrator/free" -ORCH_PIN_SHA = "30c6d71680e659f25a0a433d4726ad0d437f9757" +ORCH_PIN_SHA = "0adca4703df67f8f31d3ea5b04a1e07ed775dd6c" def _read(path: Path) -> str: