From 01766f33ccd315610592078843fce9061ab56287 Mon Sep 17 00:00:00 2001 From: Scout Date: Sun, 27 Sep 2026 01:13:39 -0400 Subject: [PATCH 1/8] Landscape-sweep templates: OpenSSF Scorecard maintenance rule; licenses never a refutation reason The sweep's templates change in two ways: - "Maintained upstream with 2025-2026 activity" becomes the OpenSSF Scorecard Maintained check. A stale repository (archived, or without a default-branch commit in the last 90 days) is excluded whatever its stars. A repository under 90 days old is flagged as too new to assess. An old tag on an active branch is not stale; pin the commit instead. - Licenses are recorded as information only and are never a reason to exclude, refute or rank down a candidate. This follows the operator's 2026-09-26 decision; the fit refuter was still told to refute non-commercial or unclear licenses. The template-hash change detector moves to PROMPTS_SHA256_CURRENT (dc5ffcb6...). The 2026-09-26 value stays as that run's recorded hash. A new test checks both rules are in the templates. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012qwXNtFyUhY5x2dtrm7tG5 --- manifests/evidence.json | 8 +++---- tests/test_landscape_sweep_harness.py | 22 ++++++++++++++----- .../landscape-sweep/templates.json | 4 ++-- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/manifests/evidence.json b/manifests/evidence.json index 71a30448b..9ff0092ee 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -38478,8 +38478,8 @@ }, { "path": "tests/test_landscape_sweep_harness.py", - "sha256": "0b5d0b94d14aad9db95a31f2f7bd015515fb26f2abe74a60254dbdd6244a8808", - "bytes": 146176 + "sha256": "2ce3f4105d76247e62fb0fd740bf12a544f90d77da63a6671f295555225b652b", + "bytes": 147126 }, { "path": "tests/test_lane_packets.py", @@ -39018,8 +39018,8 @@ }, { "path": "tools/sota-convergence/landscape-sweep/templates.json", - "sha256": "8ed97f3480daff81967d19ec31857e3e71a1407f486c3c35c9144ce8bc6a6136", - "bytes": 8614 + "sha256": "f67a0197a97fb4095ea87fad708412c12e32a1763cbd295bba5c6dc1bdcca7c3", + "bytes": 9156 }, { "path": "tools/sota-convergence/landscape-sweep/usage_record.py", diff --git a/tests/test_landscape_sweep_harness.py b/tests/test_landscape_sweep_harness.py index 05c1ab057..2fe26df6f 100644 --- a/tests/test_landscape_sweep_harness.py +++ b/tests/test_landscape_sweep_harness.py @@ -37,9 +37,14 @@ SHELLCHECK = shutil.which("shellcheck") BASH32 = os.environ.get("BASH32_BINARY") if os.environ.get("BASH32_BINARY") and os.access( os.environ["BASH32_BINARY"], os.X_OK) else None -# prompts_sha256 of the 2026-09-26 run (its templates, sha256 of json.dumps(T, sort_keys=True, ensure_ascii=False)). -# A change detector: templates.json filled with that run's values must give it. An intended template edit changes -# every later run's prompts_sha256; update this test with it (the 2026-09-26 value stays in that run's record). +# A change detector: templates.json filled with the 2026-09-26 run's values (date, layer count, skills date) must give +# PROMPTS_SHA256_CURRENT, the sha256 of json.dumps(T, sort_keys=True, ensure_ascii=False). An intended template edit +# changes every later run's prompts_sha256; update PROMPTS_SHA256_CURRENT with it. +# 2026-09-27: the maintenance rule follows the OpenSSF Scorecard Maintained check, and licenses are information only +# (never a refutation reason), per the operator's 2026-09-26/27 decisions. +PROMPTS_SHA256_CURRENT = "dc5ffcb6c58fb5de13adcf1f2ccac84b0f051a821ce89638ce8b566ebe938953" +# The 2026-09-26 run's own value, kept in that run's record (evidence/artifacts/landscape-sweep-20260926/README.md); +# fixtures below use it as a historical run's recorded prompts_sha256. PROMPTS_SHA256_20260926 = "3adfbed7a83e85da3fd7951032e1fa3a579101772a47b211580065c6b42618d4" REQ, PLAT = "a" * 64, "b" * 64 LIMIT_TEXT = ("You’ve hit your usage limit. Visit https://chatgpt.com/codex/settings/usage to purchase more " @@ -237,9 +242,16 @@ def child_usage_raw(run_id, labels, status="complete", transcript_root="/home/ex class TemplateTests(unittest.TestCase): - def test_filled_templates_reproduce_the_20260926_prompts_sha256(self): + def test_filled_templates_match_the_current_prompts_sha256(self): frozen = filled_templates("2026-09-26", 32, "2026-09-25") - self.assertEqual(sweep_common.prompts_sha256(frozen), PROMPTS_SHA256_20260926) + self.assertEqual(sweep_common.prompts_sha256(frozen), PROMPTS_SHA256_CURRENT) + + def test_templates_never_refute_on_license_and_name_the_maintenance_rule(self): + templates = json.loads((HARNESS / "templates.json").read_text()) + self.assertIn("OpenSSF Scorecard Maintained check", templates["common"]) + self.assertIn("never a reason to exclude, refute or rank down", templates["common"]) + self.assertNotIn("license is non-commercial", templates["fit"]) + self.assertIn("it is stale under the selection principles' maintenance rule", templates["fit"]) def test_only_per_call_placeholders_remain_after_filling(self): frozen = filled_templates() diff --git a/tools/sota-convergence/landscape-sweep/templates.json b/tools/sota-convergence/landscape-sweep/templates.json index e082efc0d..18755ce7d 100644 --- a/tools/sota-convergence/landscape-sweep/templates.json +++ b/tools/sota-convergence/landscape-sweep/templates.json @@ -1,8 +1,8 @@ { - "common": "Date: <>. You are one worker in a saturation sweep of the public repository seathatflowsinourveins/native-agent-stack: a portable reference stack of native agent tooling (Claude Code, Codex, MCP, memory/RAG, token efficiency, CI/supply chain, observability) whose north star is US-equities research and historical simulation (NautilusTrader 2.0.0rc5 with an IBKR destination and a separate Alpaca adapter; LEAN as comparison engine), followed by independently qualified paper trading. Live trading is out of scope.\nHosts: linux-wsl2-x86_64 GPU workstation (Threadripper PRO 5975WX 48 CPUs, 102 GiB RAM, RTX 4090 24 GB, Ada: FP8 native, NVFP4 not native) and macos-arm64 Macs (M5 Pro, 24 GB and 64 GB).\nSelection principles: native accounts and supported upstream installation; maintained upstream with 2025-2026 activity; OSI or clearly usable license (flag non-commercial, custom or unclear terms); no paid SaaS or new credentials unless a demonstrated gap requires it; local-first and privacy-filtered; evidence over popularity (stars and install counts are never evidence of fit).\nRules: primary sources only (GitHub API/pages for releases, tags, stars, pushed_at, archived, license, renames; official docs and changelogs; Hugging Face model cards). Cite the URL for every fact. Read-only: never install, clone-and-run, or execute candidate code; never read or print credentials. Canonical repository form: https://github.com// (follow renames to the current owner). Answer only in the required JSON.\nSkills (installed from adoption/skills/manifest.json, the pinned <> skills trial): use the ones that fit your role and record every skill you actually used in skills_used. Discovery: search-first (survey maintained upstream options before judging) and iterative-retrieval (staged, bounded retrieval). Refutation: verification-before-completion (evidence before any verdict), supply-chain-risk-auditor (maintenance, ownership, advisories, archived/renamed upstreams) and fp-check (verify a claim before calling it true or false). Layer-specific where relevant: agentic-actions-auditor (CI and GitHub automation), security-threat-model / codeql / semgrep / sarif-parsing (security and supply chain), property-based-testing (quality evaluation), mcp-builder (MCP surfaces), modern-python (Python tooling), agent-browser (JavaScript-heavy pages you cannot read otherwise). In Claude Code invoke a skill with the Skill tool; in Codex mention it as $skill-name. Do not load skills you do not use.\nMerit, not incumbency: the current winners are provisional. Being the current winner, the retained control (for example ai-memory in durable-memory is recorded as the retained control, not a measured best), popular, or already installed is NOT evidence of quality. Judge every repository, winners included, on current evidence: measured quality on this requirement (benchmarks, retained comparisons), SOTA-ness (recent releases, capability), maintenance and supply-chain health, license and platform fit. A candidate that is plausibly better than a winner on such evidence has a demonstrated gap. Report in notes any winner whose standing rests on incumbency rather than evidence.", + "common": "Date: <>. You are one worker in a saturation sweep of the public repository seathatflowsinourveins/native-agent-stack: a portable reference stack of native agent tooling (Claude Code, Codex, MCP, memory/RAG, token efficiency, CI/supply chain, observability) whose north star is US-equities research and historical simulation (NautilusTrader 2.0.0rc5 with an IBKR destination and a separate Alpaca adapter; LEAN as comparison engine), followed by independently qualified paper trading. Live trading is out of scope.\nHosts: linux-wsl2-x86_64 GPU workstation (Threadripper PRO 5975WX 48 CPUs, 102 GiB RAM, RTX 4090 24 GB, Ada: FP8 native, NVFP4 not native) and macos-arm64 Macs (M5 Pro, 24 GB and 64 GB).\nSelection principles: native accounts and supported upstream installation; maintained upstream per the OpenSSF Scorecard Maintained check (https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained): a stale repository, meaning archived or without a default-branch commit in the last 90 days, is excluded whatever its stars; a repository younger than 90 days is too new to assess and is flagged; an old release tag on an actively committed branch is not stale (pin the commit when a needed fix exists only on the branch); licenses are recorded as information only and are never a reason to exclude, refute or rank down a candidate (the operator's private experimental environment); no paid SaaS or new credentials unless a demonstrated gap requires it; local-first and privacy-filtered; evidence over popularity (stars and install counts are never evidence of fit).\nRules: primary sources only (GitHub API/pages for releases, tags, stars, pushed_at, archived, license, renames; official docs and changelogs; Hugging Face model cards). Cite the URL for every fact. Read-only: never install, clone-and-run, or execute candidate code; never read or print credentials. Canonical repository form: https://github.com// (follow renames to the current owner). Answer only in the required JSON.\nSkills (installed from adoption/skills/manifest.json, the pinned <> skills trial): use the ones that fit your role and record every skill you actually used in skills_used. Discovery: search-first (survey maintained upstream options before judging) and iterative-retrieval (staged, bounded retrieval). Refutation: verification-before-completion (evidence before any verdict), supply-chain-risk-auditor (maintenance, ownership, advisories, archived/renamed upstreams) and fp-check (verify a claim before calling it true or false). Layer-specific where relevant: agentic-actions-auditor (CI and GitHub automation), security-threat-model / codeql / semgrep / sarif-parsing (security and supply chain), property-based-testing (quality evaluation), mcp-builder (MCP surfaces), modern-python (Python tooling), agent-browser (JavaScript-heavy pages you cannot read otherwise). In Claude Code invoke a skill with the Skill tool; in Codex mention it as $skill-name. Do not load skills you do not use.\nMerit, not incumbency: the current winners are provisional. Being the current winner, the retained control (for example ai-memory in durable-memory is recorded as the retained control, not a measured best), popular, or already installed is NOT evidence of quality. Judge every repository, winners included, on current evidence: measured quality on this requirement (benchmarks, retained comparisons), SOTA-ness (recent releases, capability), maintenance and supply-chain health, license and platform fit. A candidate that is plausibly better than a winner on such evidence has a demonstrated gap. Report in notes any winner whose standing rests on incumbency rather than evidence.", "discover": "ROLE: discovery researcher for ONE layer.\nLayer input (requirement, current winners and pins, alternatives, upstream drift, open gaps, the previous sweep's survivors/refuted, repositories already known, and seeded candidates from other sessions):\n<>\n<>\nTask: find the strongest CURRENT candidates that could beat, replace or usefully complement this layer's winners for THIS requirement on these hosts. Cover: newest releases and models (last 4 weeks especially), strongly maintained community projects, the seeded candidates (assess every seeded item that is a repository: propose it with the right label or say in notes why not), and any winner health change (archived, renamed, relicensed, major release) which you report in notes.\nPropose at most 6 repositories. Label each:\n- targeted_candidate: worth a bounded native trial against the winner now (a concrete gap and a feasible trial on these hosts).\n- keep_but_compare: credible alternative to compare when the requirement or evidence changes.\n- not_adopted: checked and not worth pursuing (give the decisive reason) -- use this for a prominent/seeded repository you rejected so the rejection is recorded.\nFor each: repository; proposed_label; demonstrated_gap (the specific requirement gap versus the current winners, concrete, not generic praise); comparison_that_would_overturn (the measurable comparison that would decide it); evidence (URLs, each with what it shows); upstream_now (latest_release tag, released_at date, stars, pushed_at, archived, license -- from the GitHub API or page; null when unknowable); source (how you found it).\nRecord your call counts in calls (web_search, web_fetch, gh_api). Put winner-health observations and any seeded item you did not propose in notes. Set layer_id to <>.\nResearch budget (hard): at most 12 web searches, 8 page fetches and 40 GitHub API calls for this layer; stop researching when the budget is spent and answer with what you verified.", "facts": "ROLE: facts/identity refuter for ONE layer. Assume every proposal below may contain a false or unsupported fact.\nLayer: <> -- requirement: <>\nProposals (JSON):\n<>\nFor EACH proposal check with primary sources (GitHub API/pages via gh api or fetch; official docs): the repository exists at that canonical URL (or its rename), is not archived or abandoned contrary to the claim, and the license, latest release, release date, stars/activity and every cited evidence URL actually say what the proposal claims. Vote refuted=true if any material factual claim is false, unsupported by its cited source, or unverifiable; refuted=false only when you verified the facts. Default to refuted=true when uncertain. Give confidence 0..1, short reasoning, and the refs (URLs) you checked. One vote per proposal, same repository string. Set role to facts and layer_id to <>.\nVerification budget (hard): at most 8 web searches, 10 page fetches and 30 GitHub API calls in total for all proposals.", - "fit": "ROLE: fit/standing refuter for ONE layer. Assume every proposal below is wrong for this repository and try to break it.\nLayer input:\n<>\nProposals (JSON):\n<>\nFor EACH proposal vote refuted=true if ANY holds: no concrete, evidence-backed gap versus the current winners for THIS requirement (a plausible advantage in measured quality, SOTA-ness, maintenance or fit counts; incumbency does not count for the winner); it duplicates a winner or an already-known alternative without new evidence; it does not support linux x86_64/WSL2 or macOS arm64 where the layer needs it; it is infeasible on the hosts (e.g. needs more than 24 GB VRAM or more RAM than available, or NVFP4-only weights on Ada); its license is non-commercial, custom-restrictive or unclear for this use; it requires a paid SaaS or new credentials without a gap that justifies it; it conflicts with the selection principles; or its label is wrong (a targeted_candidate with no feasible bounded trial, or a not_adopted whose rejection reason is itself wrong). Vote refuted=false only when the proposal and its label hold up. Default to refuted=true when uncertain. Give confidence 0..1, reasoning, refs. One vote per proposal, same repository string. Set role to fit and layer_id to <>.\nVerification budget (hard): at most 8 web searches, 10 page fetches and 30 GitHub API calls in total for all proposals.", + "fit": "ROLE: fit/standing refuter for ONE layer. Assume every proposal below is wrong for this repository and try to break it.\nLayer input:\n<>\nProposals (JSON):\n<>\nFor EACH proposal vote refuted=true if ANY holds: no concrete, evidence-backed gap versus the current winners for THIS requirement (a plausible advantage in measured quality, SOTA-ness, maintenance or fit counts; incumbency does not count for the winner); it duplicates a winner or an already-known alternative without new evidence; it does not support linux x86_64/WSL2 or macOS arm64 where the layer needs it; it is infeasible on the hosts (e.g. needs more than 24 GB VRAM or more RAM than available, or NVFP4-only weights on Ada); it is stale under the selection principles' maintenance rule (archived, or no default-branch commit in the last 90 days); it requires a paid SaaS or new credentials without a gap that justifies it; it conflicts with the selection principles; or its label is wrong (a targeted_candidate with no feasible bounded trial, or a not_adopted whose rejection reason is itself wrong). Vote refuted=false only when the proposal and its label hold up. Default to refuted=true when uncertain. Give confidence 0..1, reasoning, refs. One vote per proposal, same repository string. Set role to fit and layer_id to <>.\nVerification budget (hard): at most 8 web searches, 10 page fetches and 30 GitHub API calls in total for all proposals.", "critic": "ROLE: completeness critic for a <>-layer saturation sweep. Below is a compact summary of every layer: requirement, winners, what was proposed and what survived both refuters.\n<>\nFind at most 8 layers where discovery most likely MISSED a strong current candidate class (a newest release or model from the last 4 weeks, a whole category of tools, a platform-specific option for linux-wsl2 RTX 4090 or macOS arm64, a north-star research or simulation runtime, a security or supply-chain option). For each give the layer_id, the reason, and 2-5 concrete search directions. Also list general observations (e.g. systematic bias). Do not re-propose candidates already proposed.", "followup": "FOLLOW-UP ROUND. The completeness critic flagged this layer: <>\nSearch directions: <>\nAlready proposed in the first round (do not re-propose): <>" } From 65cb1e8357afb500a2366a8d819b076efa4ce2b4 Mon Sep 17 00:00:00 2001 From: Scout Date: Sun, 27 Sep 2026 01:38:15 -0400 Subject: [PATCH 2/8] #385 review repair: drop license from the shared merit criteria; state the maintenance rule precisely Independent review (headless Sonnet 5, effort max): needs_changes, 1 major and 4 minor findings. - major: the shared common template, which reaches every role including the fit refuter, still judged "license and platform fit". It now reads "and platform fit (license is information only)". - minor: the rule is now "derived from" the Scorecard Maintained check and states Scorecard's actual criteria (archived is lowest; a 90-day window; top score needs a commit per week). This rule excludes only the zero-activity end. - minor: the commit fact is established with gh api .../commits?since=<90 days ago>, not pushed_at (any branch). - minor: a repository under 90 days old says "TOO NEW TO ASSESS (<90 days)" at the start of its demonstrated_gap. - minor: docs/harness-defaults.md's anti-pattern row no longer calls the template change pending. The regression test now scans every template for license-as-criterion phrases. PROMPTS_SHA256_CURRENT is updated to 0e443585. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012qwXNtFyUhY5x2dtrm7tG5 --- docs/harness-defaults.md | 2 +- manifests/evidence.json | 12 +++++----- tests/test_landscape_sweep_harness.py | 23 +++++++++++++++---- .../landscape-sweep/templates.json | 2 +- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/docs/harness-defaults.md b/docs/harness-defaults.md index c26dcb8f9..fa7a36664 100644 --- a/docs/harness-defaults.md +++ b/docs/harness-defaults.md @@ -96,7 +96,7 @@ When a claim or action proves wrong, correct it where it was relayed and record | 2026-09-26 | Trusting a flag's name instead of the request it sends | Against a local stand-in provider (cases W1 to W3 on Codex 0.155.1 and 0.157.1, local integration evidence), `codex --search exec` sent `external_web_access: false` (Codex's cached web index), the same as no flag, and only `-c web_search="live"` sent `true`; OpenAI parity rests on source reading, and a live `-c web_search="live"` search seen on OpenAI was not retained. That the landscape sweep's GPT-6 lanes, started with `codex --search exec`, searched a cached index is therefore an inference | Pass `-c web_search="live"` when a lane needs current upstream facts, and verify a flag by the request or configuration the tool actually sends | `evidence/artifacts/landscape-sweep-20260926/README.md` (cases W1 to W3); `docs/decisions/2026-09-26-workstation-tool-refresh.md`; `tools/sota-convergence/landscape-sweep/codex_job.py` still passes `--search` at this base, with its fix pending | | 2026-09-26 | Assuming each worker has its own tool budget | Claude Code allows one session at most `CLAUDE_CODE_MAX_WEB_SEARCHES_PER_SESSION` WebSearch calls (default 200), counted across the coordinator, every subagent and every workflow child; the sweep reached 200, and 46 capped calls in 30 workers returned no results | Count the session's searches before a search-heavy lane, start such a lane from a fresh or headless session with the variable raised at launch, and check workers for capped calls before using their results | `child-usage.mjs` reports each child's WebSearch and capped calls; `evidence/artifacts/landscape-sweep-20260926/README.md`; `.claude/settings.json` does not set the variable at this base | | 2026-09-26 | Treating a tool's refusal as an upstream bug before checking upstream | Context Mode refusals were drafted as upstream issues, although `ctx_execute_file` refusing paths outside the project root is intended: the pinned source names it the fix for mksglu/context-mode#852 and gives a host `permissions.allow` `Read()` rule as the escape hatch (`src/server.ts` lines 1165-1200 at context-mode `v1.0.169`), and globs anchored with `//` or `~/` do not match (open mksglu/context-mode#1075) | Run the tool's own diagnostics (`ctx_doctor`), read the pinned source and the upstream issues and releases after the pin, reproduce on upstream's recommended install, and fix our install or wiring; never file upstream | This section's check order; no mechanical check | -| 2026-09-26 | Gating candidates on license | The landscape sweep's frozen fit template refuted a proposal whose license was "non-commercial, custom-restrictive or unclear"; a GPT-6 fit re-vote on 2026-09-26, whose output the verdict wave will publish, conceded a memory candidate's measured retrieval advantage and still refuted it on that gate | Record a license as information only and decide on measured quality, security and maintenance head-to-head; flag license-based refutations for re-decision instead of flipping recorded votes | This section; the sweep template change is pending | +| 2026-09-26 | Gating candidates on license | The landscape sweep's frozen fit template refuted a proposal whose license was "non-commercial, custom-restrictive or unclear"; a GPT-6 fit re-vote on 2026-09-26, whose output the verdict wave will publish, conceded a memory candidate's measured retrieval advantage and still refuted it on that gate | Record a license as information only and decide on measured quality, security and maintenance head-to-head; flag license-based refutations for re-decision instead of flipping recorded votes | This section; the sweep templates changed in #385 (common and fit) | | 2026-09-26 | Disabling capabilities to save context instead of routing them | Never-used skills were set to `skillOverrides: "off"`, which also blocks an agent's `skills:` preload: headless `claude -p --agents` probes on 2.1.283 preloaded a `name-only` skill in full and an `off` or `user-invocable-only` skill not at all, as the sub-agents docs say (preloading draws from the skills Claude can invoke) | Keep on-demand skills `name-only`, which stays preloadable, and preload each role's small core skills through agent frontmatter `skills:`; use `user-invocable-only` only for a skill that no agent preloads; measure with `/skill-doctor` and per-child skill calls | [Probe record](decisions/2026-09-25-skills-trial-and-usage.md#addendum-2026-09-26-listing-state-and-agent-preload) | | 2026-09-26 | Paying for a keyed web API where a free native lane does the job | New research calls went to a paid, keyed search API although Claude Code WebSearch/WebFetch, Context Mode fetch and Codex live search were available | Use the free native lanes first and adopt keyless tools only through a measured bake-off | The web paragraph above | | 2026-09-26 | Treating scratch shell assignments as private | Exported `id`, `inherited`, `kept`, `rest`, `entry`, `key` and `value` reached the launcher child with changed values in a failing-first reproduction | Prefix scratch names and isolate their assignments in a subshell; verify the child environment and exact argv, including empty arguments and newlines | `LauncherTests.test_inherited_scratch_environment_and_exact_argv_reach_child` in [writer identity tests](../tests/test_observability_writer_identity.py); [POSIX and SDK behavior](../observability/collector/README.md#writer-identity-and-counter-integrity) | diff --git a/manifests/evidence.json b/manifests/evidence.json index 9ff0092ee..03c4009c3 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -13708,8 +13708,8 @@ }, { "path": "docs/harness-defaults.md", - "sha256": "30c6a0769637d7bde5d7f066aa557b10fd6c571a4bc47ac95ac206d6d660bea9", - "bytes": 29153 + "sha256": "074e054a5fd931680bd50be81d436f21eff6233357dbaa75c3627958c079b515", + "bytes": 29169 }, { "path": "docs/harness-rules-convergence-20260922.md", @@ -38478,8 +38478,8 @@ }, { "path": "tests/test_landscape_sweep_harness.py", - "sha256": "2ce3f4105d76247e62fb0fd740bf12a544f90d77da63a6671f295555225b652b", - "bytes": 147126 + "sha256": "882676248a11e1f144dd72d3eda28cd00da85f7a009c7867d03854713710040c", + "bytes": 148106 }, { "path": "tests/test_lane_packets.py", @@ -39018,8 +39018,8 @@ }, { "path": "tools/sota-convergence/landscape-sweep/templates.json", - "sha256": "f67a0197a97fb4095ea87fad708412c12e32a1763cbd295bba5c6dc1bdcca7c3", - "bytes": 9156 + "sha256": "7b7ccf702ef9c5ff477efa66a0efac02315593b8beb8572b189b086dc776fbb5", + "bytes": 9676 }, { "path": "tools/sota-convergence/landscape-sweep/usage_record.py", diff --git a/tests/test_landscape_sweep_harness.py b/tests/test_landscape_sweep_harness.py index 2fe26df6f..5470c5dad 100644 --- a/tests/test_landscape_sweep_harness.py +++ b/tests/test_landscape_sweep_harness.py @@ -42,7 +42,7 @@ # changes every later run's prompts_sha256; update PROMPTS_SHA256_CURRENT with it. # 2026-09-27: the maintenance rule follows the OpenSSF Scorecard Maintained check, and licenses are information only # (never a refutation reason), per the operator's 2026-09-26/27 decisions. -PROMPTS_SHA256_CURRENT = "dc5ffcb6c58fb5de13adcf1f2ccac84b0f051a821ce89638ce8b566ebe938953" +PROMPTS_SHA256_CURRENT = "0e4435853c80a29eca8a699a2449a095dfb0efd39ca9c89eba0cefd530a54367" # The 2026-09-26 run's own value, kept in that run's record (evidence/artifacts/landscape-sweep-20260926/README.md); # fixtures below use it as a historical run's recorded prompts_sha256. PROMPTS_SHA256_20260926 = "3adfbed7a83e85da3fd7951032e1fa3a579101772a47b211580065c6b42618d4" @@ -248,10 +248,23 @@ def test_filled_templates_match_the_current_prompts_sha256(self): def test_templates_never_refute_on_license_and_name_the_maintenance_rule(self): templates = json.loads((HARNESS / "templates.json").read_text()) - self.assertIn("OpenSSF Scorecard Maintained check", templates["common"]) - self.assertIn("never a reason to exclude, refute or rank down", templates["common"]) - self.assertNotIn("license is non-commercial", templates["fit"]) - self.assertIn("it is stale under the selection principles' maintenance rule", templates["fit"]) + common, fit = templates["common"], templates["fit"] + # Licenses: information only, in the selection principles and in the merit criteria every role receives. + self.assertIn("never a reason to exclude, refute or rank down", common) + self.assertIn("and platform fit (license is information only)", common) + self.assertNotIn("license and platform fit", common) + self.assertNotIn("OSI or clearly usable license", common) + self.assertNotIn("license is non-commercial", fit) + # No role other than the facts refuter's accuracy check may treat a license as a criterion. + for key, text in templates.items(): + for phrase in ("license is non-commercial", "restrictive license", "unclear license", "usable license"): + self.assertNotIn(phrase, text, f"{key} still uses a license as a criterion: {phrase!r}") + # Maintenance: derived from the Scorecard check, with its evidence command and a flag destination. + self.assertIn("derived from the OpenSSF Scorecard Maintained check", common) + self.assertIn("commits?since=", common) + self.assertIn("not from pushed_at", common) + self.assertIn("TOO NEW TO ASSESS (<90 days)", common) + self.assertIn("it is stale under the selection principles' maintenance rule", fit) def test_only_per_call_placeholders_remain_after_filling(self): frozen = filled_templates() diff --git a/tools/sota-convergence/landscape-sweep/templates.json b/tools/sota-convergence/landscape-sweep/templates.json index 18755ce7d..7956176bf 100644 --- a/tools/sota-convergence/landscape-sweep/templates.json +++ b/tools/sota-convergence/landscape-sweep/templates.json @@ -1,5 +1,5 @@ { - "common": "Date: <>. You are one worker in a saturation sweep of the public repository seathatflowsinourveins/native-agent-stack: a portable reference stack of native agent tooling (Claude Code, Codex, MCP, memory/RAG, token efficiency, CI/supply chain, observability) whose north star is US-equities research and historical simulation (NautilusTrader 2.0.0rc5 with an IBKR destination and a separate Alpaca adapter; LEAN as comparison engine), followed by independently qualified paper trading. Live trading is out of scope.\nHosts: linux-wsl2-x86_64 GPU workstation (Threadripper PRO 5975WX 48 CPUs, 102 GiB RAM, RTX 4090 24 GB, Ada: FP8 native, NVFP4 not native) and macos-arm64 Macs (M5 Pro, 24 GB and 64 GB).\nSelection principles: native accounts and supported upstream installation; maintained upstream per the OpenSSF Scorecard Maintained check (https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained): a stale repository, meaning archived or without a default-branch commit in the last 90 days, is excluded whatever its stars; a repository younger than 90 days is too new to assess and is flagged; an old release tag on an actively committed branch is not stale (pin the commit when a needed fix exists only on the branch); licenses are recorded as information only and are never a reason to exclude, refute or rank down a candidate (the operator's private experimental environment); no paid SaaS or new credentials unless a demonstrated gap requires it; local-first and privacy-filtered; evidence over popularity (stars and install counts are never evidence of fit).\nRules: primary sources only (GitHub API/pages for releases, tags, stars, pushed_at, archived, license, renames; official docs and changelogs; Hugging Face model cards). Cite the URL for every fact. Read-only: never install, clone-and-run, or execute candidate code; never read or print credentials. Canonical repository form: https://github.com// (follow renames to the current owner). Answer only in the required JSON.\nSkills (installed from adoption/skills/manifest.json, the pinned <> skills trial): use the ones that fit your role and record every skill you actually used in skills_used. Discovery: search-first (survey maintained upstream options before judging) and iterative-retrieval (staged, bounded retrieval). Refutation: verification-before-completion (evidence before any verdict), supply-chain-risk-auditor (maintenance, ownership, advisories, archived/renamed upstreams) and fp-check (verify a claim before calling it true or false). Layer-specific where relevant: agentic-actions-auditor (CI and GitHub automation), security-threat-model / codeql / semgrep / sarif-parsing (security and supply chain), property-based-testing (quality evaluation), mcp-builder (MCP surfaces), modern-python (Python tooling), agent-browser (JavaScript-heavy pages you cannot read otherwise). In Claude Code invoke a skill with the Skill tool; in Codex mention it as $skill-name. Do not load skills you do not use.\nMerit, not incumbency: the current winners are provisional. Being the current winner, the retained control (for example ai-memory in durable-memory is recorded as the retained control, not a measured best), popular, or already installed is NOT evidence of quality. Judge every repository, winners included, on current evidence: measured quality on this requirement (benchmarks, retained comparisons), SOTA-ness (recent releases, capability), maintenance and supply-chain health, license and platform fit. A candidate that is plausibly better than a winner on such evidence has a demonstrated gap. Report in notes any winner whose standing rests on incumbency rather than evidence.", + "common": "Date: <>. You are one worker in a saturation sweep of the public repository seathatflowsinourveins/native-agent-stack: a portable reference stack of native agent tooling (Claude Code, Codex, MCP, memory/RAG, token efficiency, CI/supply chain, observability) whose north star is US-equities research and historical simulation (NautilusTrader 2.0.0rc5 with an IBKR destination and a separate Alpaca adapter; LEAN as comparison engine), followed by independently qualified paper trading. Live trading is out of scope.\nHosts: linux-wsl2-x86_64 GPU workstation (Threadripper PRO 5975WX 48 CPUs, 102 GiB RAM, RTX 4090 24 GB, Ada: FP8 native, NVFP4 not native) and macos-arm64 Macs (M5 Pro, 24 GB and 64 GB).\nSelection principles: native accounts and supported upstream installation; maintained upstream, by a rule derived from the OpenSSF Scorecard Maintained check (https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained), which scores an archived repository lowest, uses a 90-day window and gives its top score only for at least one commit per week: a repository is stale, and excluded whatever its stars, when it is archived or has no commit on its default branch in the last 90 days (establish this with gh api repos///commits?since=&per_page=1, not from pushed_at, which counts any branch); this rule excludes only the zero-activity end, so record lower activity as a maintenance fact without excluding; a repository created less than 90 days ago is too new to assess and must say so at the start of its demonstrated_gap (\"TOO NEW TO ASSESS (<90 days): ...\"); an old release tag on an actively committed branch is not stale (pin the commit when a needed fix exists only on the branch); licenses are recorded as information only and are never a reason to exclude, refute or rank down a candidate (the operator's private experimental environment); no paid SaaS or new credentials unless a demonstrated gap requires it; local-first and privacy-filtered; evidence over popularity (stars and install counts are never evidence of fit).\nRules: primary sources only (GitHub API/pages for releases, tags, stars, pushed_at, archived, license, renames; official docs and changelogs; Hugging Face model cards). Cite the URL for every fact. Read-only: never install, clone-and-run, or execute candidate code; never read or print credentials. Canonical repository form: https://github.com// (follow renames to the current owner). Answer only in the required JSON.\nSkills (installed from adoption/skills/manifest.json, the pinned <> skills trial): use the ones that fit your role and record every skill you actually used in skills_used. Discovery: search-first (survey maintained upstream options before judging) and iterative-retrieval (staged, bounded retrieval). Refutation: verification-before-completion (evidence before any verdict), supply-chain-risk-auditor (maintenance, ownership, advisories, archived/renamed upstreams) and fp-check (verify a claim before calling it true or false). Layer-specific where relevant: agentic-actions-auditor (CI and GitHub automation), security-threat-model / codeql / semgrep / sarif-parsing (security and supply chain), property-based-testing (quality evaluation), mcp-builder (MCP surfaces), modern-python (Python tooling), agent-browser (JavaScript-heavy pages you cannot read otherwise). In Claude Code invoke a skill with the Skill tool; in Codex mention it as $skill-name. Do not load skills you do not use.\nMerit, not incumbency: the current winners are provisional. Being the current winner, the retained control (for example ai-memory in durable-memory is recorded as the retained control, not a measured best), popular, or already installed is NOT evidence of quality. Judge every repository, winners included, on current evidence: measured quality on this requirement (benchmarks, retained comparisons), SOTA-ness (recent releases, capability), maintenance and supply-chain health, and platform fit (license is information only). A candidate that is plausibly better than a winner on such evidence has a demonstrated gap. Report in notes any winner whose standing rests on incumbency rather than evidence.", "discover": "ROLE: discovery researcher for ONE layer.\nLayer input (requirement, current winners and pins, alternatives, upstream drift, open gaps, the previous sweep's survivors/refuted, repositories already known, and seeded candidates from other sessions):\n<>\n<>\nTask: find the strongest CURRENT candidates that could beat, replace or usefully complement this layer's winners for THIS requirement on these hosts. Cover: newest releases and models (last 4 weeks especially), strongly maintained community projects, the seeded candidates (assess every seeded item that is a repository: propose it with the right label or say in notes why not), and any winner health change (archived, renamed, relicensed, major release) which you report in notes.\nPropose at most 6 repositories. Label each:\n- targeted_candidate: worth a bounded native trial against the winner now (a concrete gap and a feasible trial on these hosts).\n- keep_but_compare: credible alternative to compare when the requirement or evidence changes.\n- not_adopted: checked and not worth pursuing (give the decisive reason) -- use this for a prominent/seeded repository you rejected so the rejection is recorded.\nFor each: repository; proposed_label; demonstrated_gap (the specific requirement gap versus the current winners, concrete, not generic praise); comparison_that_would_overturn (the measurable comparison that would decide it); evidence (URLs, each with what it shows); upstream_now (latest_release tag, released_at date, stars, pushed_at, archived, license -- from the GitHub API or page; null when unknowable); source (how you found it).\nRecord your call counts in calls (web_search, web_fetch, gh_api). Put winner-health observations and any seeded item you did not propose in notes. Set layer_id to <>.\nResearch budget (hard): at most 12 web searches, 8 page fetches and 40 GitHub API calls for this layer; stop researching when the budget is spent and answer with what you verified.", "facts": "ROLE: facts/identity refuter for ONE layer. Assume every proposal below may contain a false or unsupported fact.\nLayer: <> -- requirement: <>\nProposals (JSON):\n<>\nFor EACH proposal check with primary sources (GitHub API/pages via gh api or fetch; official docs): the repository exists at that canonical URL (or its rename), is not archived or abandoned contrary to the claim, and the license, latest release, release date, stars/activity and every cited evidence URL actually say what the proposal claims. Vote refuted=true if any material factual claim is false, unsupported by its cited source, or unverifiable; refuted=false only when you verified the facts. Default to refuted=true when uncertain. Give confidence 0..1, short reasoning, and the refs (URLs) you checked. One vote per proposal, same repository string. Set role to facts and layer_id to <>.\nVerification budget (hard): at most 8 web searches, 10 page fetches and 30 GitHub API calls in total for all proposals.", "fit": "ROLE: fit/standing refuter for ONE layer. Assume every proposal below is wrong for this repository and try to break it.\nLayer input:\n<>\nProposals (JSON):\n<>\nFor EACH proposal vote refuted=true if ANY holds: no concrete, evidence-backed gap versus the current winners for THIS requirement (a plausible advantage in measured quality, SOTA-ness, maintenance or fit counts; incumbency does not count for the winner); it duplicates a winner or an already-known alternative without new evidence; it does not support linux x86_64/WSL2 or macOS arm64 where the layer needs it; it is infeasible on the hosts (e.g. needs more than 24 GB VRAM or more RAM than available, or NVFP4-only weights on Ada); it is stale under the selection principles' maintenance rule (archived, or no default-branch commit in the last 90 days); it requires a paid SaaS or new credentials without a gap that justifies it; it conflicts with the selection principles; or its label is wrong (a targeted_candidate with no feasible bounded trial, or a not_adopted whose rejection reason is itself wrong). Vote refuted=false only when the proposal and its label hold up. Default to refuted=true when uncertain. Give confidence 0..1, reasoning, refs. One vote per proposal, same repository string. Set role to fit and layer_id to <>.\nVerification budget (hard): at most 8 web searches, 10 page fetches and 30 GitHub API calls in total for all proposals.", From b87a6563bdbb885cd679704b0696f502f6afacf3 Mon Sep 17 00:00:00 2001 From: Scout Date: Sun, 27 Sep 2026 01:52:31 -0400 Subject: [PATCH 3/8] #385 re-check residuals: Scorecard's weekly commit is sufficient, not necessary; REST URL alternative The independent Opus re-check agreed the major and 3 of 4 minor findings were resolved, and raised two minors. - The shared template said Scorecard "gives its top score only for at least one commit per week". It now gives the top score to at least one commit per week and partial credit for maintainers' issue activity (the Scorecard doc's sufficient condition). - The commit check also names the same https://api.github.com REST URL, for lanes without the gh CLI. The test comment says "derived from", and the anti-pattern log names the test. PROMPTS_SHA256_CURRENT is updated to a9722fec. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012qwXNtFyUhY5x2dtrm7tG5 --- docs/harness-defaults.md | 2 +- manifests/evidence.json | 12 ++++++------ tests/test_landscape_sweep_harness.py | 4 ++-- .../sota-convergence/landscape-sweep/templates.json | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/harness-defaults.md b/docs/harness-defaults.md index fa7a36664..a0d008e07 100644 --- a/docs/harness-defaults.md +++ b/docs/harness-defaults.md @@ -96,7 +96,7 @@ When a claim or action proves wrong, correct it where it was relayed and record | 2026-09-26 | Trusting a flag's name instead of the request it sends | Against a local stand-in provider (cases W1 to W3 on Codex 0.155.1 and 0.157.1, local integration evidence), `codex --search exec` sent `external_web_access: false` (Codex's cached web index), the same as no flag, and only `-c web_search="live"` sent `true`; OpenAI parity rests on source reading, and a live `-c web_search="live"` search seen on OpenAI was not retained. That the landscape sweep's GPT-6 lanes, started with `codex --search exec`, searched a cached index is therefore an inference | Pass `-c web_search="live"` when a lane needs current upstream facts, and verify a flag by the request or configuration the tool actually sends | `evidence/artifacts/landscape-sweep-20260926/README.md` (cases W1 to W3); `docs/decisions/2026-09-26-workstation-tool-refresh.md`; `tools/sota-convergence/landscape-sweep/codex_job.py` still passes `--search` at this base, with its fix pending | | 2026-09-26 | Assuming each worker has its own tool budget | Claude Code allows one session at most `CLAUDE_CODE_MAX_WEB_SEARCHES_PER_SESSION` WebSearch calls (default 200), counted across the coordinator, every subagent and every workflow child; the sweep reached 200, and 46 capped calls in 30 workers returned no results | Count the session's searches before a search-heavy lane, start such a lane from a fresh or headless session with the variable raised at launch, and check workers for capped calls before using their results | `child-usage.mjs` reports each child's WebSearch and capped calls; `evidence/artifacts/landscape-sweep-20260926/README.md`; `.claude/settings.json` does not set the variable at this base | | 2026-09-26 | Treating a tool's refusal as an upstream bug before checking upstream | Context Mode refusals were drafted as upstream issues, although `ctx_execute_file` refusing paths outside the project root is intended: the pinned source names it the fix for mksglu/context-mode#852 and gives a host `permissions.allow` `Read()` rule as the escape hatch (`src/server.ts` lines 1165-1200 at context-mode `v1.0.169`), and globs anchored with `//` or `~/` do not match (open mksglu/context-mode#1075) | Run the tool's own diagnostics (`ctx_doctor`), read the pinned source and the upstream issues and releases after the pin, reproduce on upstream's recommended install, and fix our install or wiring; never file upstream | This section's check order; no mechanical check | -| 2026-09-26 | Gating candidates on license | The landscape sweep's frozen fit template refuted a proposal whose license was "non-commercial, custom-restrictive or unclear"; a GPT-6 fit re-vote on 2026-09-26, whose output the verdict wave will publish, conceded a memory candidate's measured retrieval advantage and still refuted it on that gate | Record a license as information only and decide on measured quality, security and maintenance head-to-head; flag license-based refutations for re-decision instead of flipping recorded votes | This section; the sweep templates changed in #385 (common and fit) | +| 2026-09-26 | Gating candidates on license | The landscape sweep's frozen fit template refuted a proposal whose license was "non-commercial, custom-restrictive or unclear"; a GPT-6 fit re-vote on 2026-09-26, whose output the verdict wave will publish, conceded a memory candidate's measured retrieval advantage and still refuted it on that gate | Record a license as information only and decide on measured quality, security and maintenance head-to-head; flag license-based refutations for re-decision instead of flipping recorded votes | This section; the sweep templates changed in #385 (common and fit), checked by `tests/test_landscape_sweep_harness.py` | | 2026-09-26 | Disabling capabilities to save context instead of routing them | Never-used skills were set to `skillOverrides: "off"`, which also blocks an agent's `skills:` preload: headless `claude -p --agents` probes on 2.1.283 preloaded a `name-only` skill in full and an `off` or `user-invocable-only` skill not at all, as the sub-agents docs say (preloading draws from the skills Claude can invoke) | Keep on-demand skills `name-only`, which stays preloadable, and preload each role's small core skills through agent frontmatter `skills:`; use `user-invocable-only` only for a skill that no agent preloads; measure with `/skill-doctor` and per-child skill calls | [Probe record](decisions/2026-09-25-skills-trial-and-usage.md#addendum-2026-09-26-listing-state-and-agent-preload) | | 2026-09-26 | Paying for a keyed web API where a free native lane does the job | New research calls went to a paid, keyed search API although Claude Code WebSearch/WebFetch, Context Mode fetch and Codex live search were available | Use the free native lanes first and adopt keyless tools only through a measured bake-off | The web paragraph above | | 2026-09-26 | Treating scratch shell assignments as private | Exported `id`, `inherited`, `kept`, `rest`, `entry`, `key` and `value` reached the launcher child with changed values in a failing-first reproduction | Prefix scratch names and isolate their assignments in a subshell; verify the child environment and exact argv, including empty arguments and newlines | `LauncherTests.test_inherited_scratch_environment_and_exact_argv_reach_child` in [writer identity tests](../tests/test_observability_writer_identity.py); [POSIX and SDK behavior](../observability/collector/README.md#writer-identity-and-counter-integrity) | diff --git a/manifests/evidence.json b/manifests/evidence.json index 03c4009c3..47ab30b76 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -13708,8 +13708,8 @@ }, { "path": "docs/harness-defaults.md", - "sha256": "074e054a5fd931680bd50be81d436f21eff6233357dbaa75c3627958c079b515", - "bytes": 29169 + "sha256": "4c2d416f56d20032e3cf1183e113a7cd3b213a0232f3dc37e9e71648c9700f93", + "bytes": 29221 }, { "path": "docs/harness-rules-convergence-20260922.md", @@ -38478,8 +38478,8 @@ }, { "path": "tests/test_landscape_sweep_harness.py", - "sha256": "882676248a11e1f144dd72d3eda28cd00da85f7a009c7867d03854713710040c", - "bytes": 148106 + "sha256": "3dd96aa5df99436cdfa61a99a2a868191da01fb84cd098540bf34b79b85ac9c1", + "bytes": 148114 }, { "path": "tests/test_lane_packets.py", @@ -39018,8 +39018,8 @@ }, { "path": "tools/sota-convergence/landscape-sweep/templates.json", - "sha256": "7b7ccf702ef9c5ff477efa66a0efac02315593b8beb8572b189b086dc776fbb5", - "bytes": 9676 + "sha256": "d039ea4b6b81af468716bdc725abd6b16b8d1796b9eab7f82b34f32dde0406cb", + "bytes": 9765 }, { "path": "tools/sota-convergence/landscape-sweep/usage_record.py", diff --git a/tests/test_landscape_sweep_harness.py b/tests/test_landscape_sweep_harness.py index 5470c5dad..84e1b7f0f 100644 --- a/tests/test_landscape_sweep_harness.py +++ b/tests/test_landscape_sweep_harness.py @@ -40,9 +40,9 @@ # A change detector: templates.json filled with the 2026-09-26 run's values (date, layer count, skills date) must give # PROMPTS_SHA256_CURRENT, the sha256 of json.dumps(T, sort_keys=True, ensure_ascii=False). An intended template edit # changes every later run's prompts_sha256; update PROMPTS_SHA256_CURRENT with it. -# 2026-09-27: the maintenance rule follows the OpenSSF Scorecard Maintained check, and licenses are information only +# 2026-09-27: the maintenance rule is derived from the OpenSSF Scorecard Maintained check, and licenses are information only # (never a refutation reason), per the operator's 2026-09-26/27 decisions. -PROMPTS_SHA256_CURRENT = "0e4435853c80a29eca8a699a2449a095dfb0efd39ca9c89eba0cefd530a54367" +PROMPTS_SHA256_CURRENT = "a9722fec777e5a5d6037be15ccda8ad208a06bba075fcb9832e49b54bcff1fb4" # The 2026-09-26 run's own value, kept in that run's record (evidence/artifacts/landscape-sweep-20260926/README.md); # fixtures below use it as a historical run's recorded prompts_sha256. PROMPTS_SHA256_20260926 = "3adfbed7a83e85da3fd7951032e1fa3a579101772a47b211580065c6b42618d4" diff --git a/tools/sota-convergence/landscape-sweep/templates.json b/tools/sota-convergence/landscape-sweep/templates.json index 7956176bf..ec9e5cd2f 100644 --- a/tools/sota-convergence/landscape-sweep/templates.json +++ b/tools/sota-convergence/landscape-sweep/templates.json @@ -1,5 +1,5 @@ { - "common": "Date: <>. You are one worker in a saturation sweep of the public repository seathatflowsinourveins/native-agent-stack: a portable reference stack of native agent tooling (Claude Code, Codex, MCP, memory/RAG, token efficiency, CI/supply chain, observability) whose north star is US-equities research and historical simulation (NautilusTrader 2.0.0rc5 with an IBKR destination and a separate Alpaca adapter; LEAN as comparison engine), followed by independently qualified paper trading. Live trading is out of scope.\nHosts: linux-wsl2-x86_64 GPU workstation (Threadripper PRO 5975WX 48 CPUs, 102 GiB RAM, RTX 4090 24 GB, Ada: FP8 native, NVFP4 not native) and macos-arm64 Macs (M5 Pro, 24 GB and 64 GB).\nSelection principles: native accounts and supported upstream installation; maintained upstream, by a rule derived from the OpenSSF Scorecard Maintained check (https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained), which scores an archived repository lowest, uses a 90-day window and gives its top score only for at least one commit per week: a repository is stale, and excluded whatever its stars, when it is archived or has no commit on its default branch in the last 90 days (establish this with gh api repos///commits?since=&per_page=1, not from pushed_at, which counts any branch); this rule excludes only the zero-activity end, so record lower activity as a maintenance fact without excluding; a repository created less than 90 days ago is too new to assess and must say so at the start of its demonstrated_gap (\"TOO NEW TO ASSESS (<90 days): ...\"); an old release tag on an actively committed branch is not stale (pin the commit when a needed fix exists only on the branch); licenses are recorded as information only and are never a reason to exclude, refute or rank down a candidate (the operator's private experimental environment); no paid SaaS or new credentials unless a demonstrated gap requires it; local-first and privacy-filtered; evidence over popularity (stars and install counts are never evidence of fit).\nRules: primary sources only (GitHub API/pages for releases, tags, stars, pushed_at, archived, license, renames; official docs and changelogs; Hugging Face model cards). Cite the URL for every fact. Read-only: never install, clone-and-run, or execute candidate code; never read or print credentials. Canonical repository form: https://github.com// (follow renames to the current owner). Answer only in the required JSON.\nSkills (installed from adoption/skills/manifest.json, the pinned <> skills trial): use the ones that fit your role and record every skill you actually used in skills_used. Discovery: search-first (survey maintained upstream options before judging) and iterative-retrieval (staged, bounded retrieval). Refutation: verification-before-completion (evidence before any verdict), supply-chain-risk-auditor (maintenance, ownership, advisories, archived/renamed upstreams) and fp-check (verify a claim before calling it true or false). Layer-specific where relevant: agentic-actions-auditor (CI and GitHub automation), security-threat-model / codeql / semgrep / sarif-parsing (security and supply chain), property-based-testing (quality evaluation), mcp-builder (MCP surfaces), modern-python (Python tooling), agent-browser (JavaScript-heavy pages you cannot read otherwise). In Claude Code invoke a skill with the Skill tool; in Codex mention it as $skill-name. Do not load skills you do not use.\nMerit, not incumbency: the current winners are provisional. Being the current winner, the retained control (for example ai-memory in durable-memory is recorded as the retained control, not a measured best), popular, or already installed is NOT evidence of quality. Judge every repository, winners included, on current evidence: measured quality on this requirement (benchmarks, retained comparisons), SOTA-ness (recent releases, capability), maintenance and supply-chain health, and platform fit (license is information only). A candidate that is plausibly better than a winner on such evidence has a demonstrated gap. Report in notes any winner whose standing rests on incumbency rather than evidence.", + "common": "Date: <>. You are one worker in a saturation sweep of the public repository seathatflowsinourveins/native-agent-stack: a portable reference stack of native agent tooling (Claude Code, Codex, MCP, memory/RAG, token efficiency, CI/supply chain, observability) whose north star is US-equities research and historical simulation (NautilusTrader 2.0.0rc5 with an IBKR destination and a separate Alpaca adapter; LEAN as comparison engine), followed by independently qualified paper trading. Live trading is out of scope.\nHosts: linux-wsl2-x86_64 GPU workstation (Threadripper PRO 5975WX 48 CPUs, 102 GiB RAM, RTX 4090 24 GB, Ada: FP8 native, NVFP4 not native) and macos-arm64 Macs (M5 Pro, 24 GB and 64 GB).\nSelection principles: native accounts and supported upstream installation; maintained upstream, by a rule derived from the OpenSSF Scorecard Maintained check (https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained), which scores an archived repository lowest, uses a 90-day window and gives its top score to at least one commit per week and partial credit for maintainers' issue activity: a repository is stale, and excluded whatever its stars, when it is archived or has no commit on its default branch in the last 90 days (establish this with gh api repos///commits?since=&per_page=1 or the same https://api.github.com REST URL, not from pushed_at, which counts any branch); this rule excludes only the zero-activity end, so record lower activity as a maintenance fact without excluding; a repository created less than 90 days ago is too new to assess and must say so at the start of its demonstrated_gap (\"TOO NEW TO ASSESS (<90 days): ...\"); an old release tag on an actively committed branch is not stale (pin the commit when a needed fix exists only on the branch); licenses are recorded as information only and are never a reason to exclude, refute or rank down a candidate (the operator's private experimental environment); no paid SaaS or new credentials unless a demonstrated gap requires it; local-first and privacy-filtered; evidence over popularity (stars and install counts are never evidence of fit).\nRules: primary sources only (GitHub API/pages for releases, tags, stars, pushed_at, archived, license, renames; official docs and changelogs; Hugging Face model cards). Cite the URL for every fact. Read-only: never install, clone-and-run, or execute candidate code; never read or print credentials. Canonical repository form: https://github.com// (follow renames to the current owner). Answer only in the required JSON.\nSkills (installed from adoption/skills/manifest.json, the pinned <> skills trial): use the ones that fit your role and record every skill you actually used in skills_used. Discovery: search-first (survey maintained upstream options before judging) and iterative-retrieval (staged, bounded retrieval). Refutation: verification-before-completion (evidence before any verdict), supply-chain-risk-auditor (maintenance, ownership, advisories, archived/renamed upstreams) and fp-check (verify a claim before calling it true or false). Layer-specific where relevant: agentic-actions-auditor (CI and GitHub automation), security-threat-model / codeql / semgrep / sarif-parsing (security and supply chain), property-based-testing (quality evaluation), mcp-builder (MCP surfaces), modern-python (Python tooling), agent-browser (JavaScript-heavy pages you cannot read otherwise). In Claude Code invoke a skill with the Skill tool; in Codex mention it as $skill-name. Do not load skills you do not use.\nMerit, not incumbency: the current winners are provisional. Being the current winner, the retained control (for example ai-memory in durable-memory is recorded as the retained control, not a measured best), popular, or already installed is NOT evidence of quality. Judge every repository, winners included, on current evidence: measured quality on this requirement (benchmarks, retained comparisons), SOTA-ness (recent releases, capability), maintenance and supply-chain health, and platform fit (license is information only). A candidate that is plausibly better than a winner on such evidence has a demonstrated gap. Report in notes any winner whose standing rests on incumbency rather than evidence.", "discover": "ROLE: discovery researcher for ONE layer.\nLayer input (requirement, current winners and pins, alternatives, upstream drift, open gaps, the previous sweep's survivors/refuted, repositories already known, and seeded candidates from other sessions):\n<>\n<>\nTask: find the strongest CURRENT candidates that could beat, replace or usefully complement this layer's winners for THIS requirement on these hosts. Cover: newest releases and models (last 4 weeks especially), strongly maintained community projects, the seeded candidates (assess every seeded item that is a repository: propose it with the right label or say in notes why not), and any winner health change (archived, renamed, relicensed, major release) which you report in notes.\nPropose at most 6 repositories. Label each:\n- targeted_candidate: worth a bounded native trial against the winner now (a concrete gap and a feasible trial on these hosts).\n- keep_but_compare: credible alternative to compare when the requirement or evidence changes.\n- not_adopted: checked and not worth pursuing (give the decisive reason) -- use this for a prominent/seeded repository you rejected so the rejection is recorded.\nFor each: repository; proposed_label; demonstrated_gap (the specific requirement gap versus the current winners, concrete, not generic praise); comparison_that_would_overturn (the measurable comparison that would decide it); evidence (URLs, each with what it shows); upstream_now (latest_release tag, released_at date, stars, pushed_at, archived, license -- from the GitHub API or page; null when unknowable); source (how you found it).\nRecord your call counts in calls (web_search, web_fetch, gh_api). Put winner-health observations and any seeded item you did not propose in notes. Set layer_id to <>.\nResearch budget (hard): at most 12 web searches, 8 page fetches and 40 GitHub API calls for this layer; stop researching when the budget is spent and answer with what you verified.", "facts": "ROLE: facts/identity refuter for ONE layer. Assume every proposal below may contain a false or unsupported fact.\nLayer: <> -- requirement: <>\nProposals (JSON):\n<>\nFor EACH proposal check with primary sources (GitHub API/pages via gh api or fetch; official docs): the repository exists at that canonical URL (or its rename), is not archived or abandoned contrary to the claim, and the license, latest release, release date, stars/activity and every cited evidence URL actually say what the proposal claims. Vote refuted=true if any material factual claim is false, unsupported by its cited source, or unverifiable; refuted=false only when you verified the facts. Default to refuted=true when uncertain. Give confidence 0..1, short reasoning, and the refs (URLs) you checked. One vote per proposal, same repository string. Set role to facts and layer_id to <>.\nVerification budget (hard): at most 8 web searches, 10 page fetches and 30 GitHub API calls in total for all proposals.", "fit": "ROLE: fit/standing refuter for ONE layer. Assume every proposal below is wrong for this repository and try to break it.\nLayer input:\n<>\nProposals (JSON):\n<>\nFor EACH proposal vote refuted=true if ANY holds: no concrete, evidence-backed gap versus the current winners for THIS requirement (a plausible advantage in measured quality, SOTA-ness, maintenance or fit counts; incumbency does not count for the winner); it duplicates a winner or an already-known alternative without new evidence; it does not support linux x86_64/WSL2 or macOS arm64 where the layer needs it; it is infeasible on the hosts (e.g. needs more than 24 GB VRAM or more RAM than available, or NVFP4-only weights on Ada); it is stale under the selection principles' maintenance rule (archived, or no default-branch commit in the last 90 days); it requires a paid SaaS or new credentials without a gap that justifies it; it conflicts with the selection principles; or its label is wrong (a targeted_candidate with no feasible bounded trial, or a not_adopted whose rejection reason is itself wrong). Vote refuted=false only when the proposal and its label hold up. Default to refuted=true when uncertain. Give confidence 0..1, reasoning, refs. One vote per proposal, same repository string. Set role to fit and layer_id to <>.\nVerification budget (hard): at most 8 web searches, 10 page fetches and 30 GitHub API calls in total for all proposals.", From 99837f2ee76353bf48c616f7611951eb24713a2a Mon Sep 17 00:00:00 2001 From: Scout Date: Sun, 27 Sep 2026 02:03:42 -0400 Subject: [PATCH 4/8] #385 GPT-6 review: unknown maintenance is never a refutation (web-only lanes cannot reach the commit evidence) GPT-6 cross-family review (gpt-6-astra, max) of b87a6563 returned needs_changes, with 1 medium finding. The web-only GPT-6 lane cannot run gh (make_prompt.py TAIL), and its web tools failed to fetch the commits API URL. Since the fit refuter defaults to refuted, maintained repositories could be refuted for unreachable evidence. - common: a lane that cannot reach either source records the commit fact as unknown and never excludes or refutes on unknown maintenance, because a missing result is pending, never refuted. - fit: staleness must be established from evidence; unknown maintenance is never a reason to refute. The test asserts both. PROMPTS_SHA256_CURRENT is updated to f64eec22. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012qwXNtFyUhY5x2dtrm7tG5 --- manifests/evidence.json | 8 ++++---- tests/test_landscape_sweep_harness.py | 5 ++++- tools/sota-convergence/landscape-sweep/templates.json | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifests/evidence.json b/manifests/evidence.json index 47ab30b76..8bd953193 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -38478,8 +38478,8 @@ }, { "path": "tests/test_landscape_sweep_harness.py", - "sha256": "3dd96aa5df99436cdfa61a99a2a868191da01fb84cd098540bf34b79b85ac9c1", - "bytes": 148114 + "sha256": "7f9c7ab0e3555563f1c26a651005cb3affad2ab4f19628bcbdf581fb800e7c17", + "bytes": 148389 }, { "path": "tests/test_lane_packets.py", @@ -39018,8 +39018,8 @@ }, { "path": "tools/sota-convergence/landscape-sweep/templates.json", - "sha256": "d039ea4b6b81af468716bdc725abd6b16b8d1796b9eab7f82b34f32dde0406cb", - "bytes": 9765 + "sha256": "56caeef02581a21ad672470718295f8d6f210d13ce4b6ee7783a59ac383ce675", + "bytes": 10060 }, { "path": "tools/sota-convergence/landscape-sweep/usage_record.py", diff --git a/tests/test_landscape_sweep_harness.py b/tests/test_landscape_sweep_harness.py index 84e1b7f0f..3bd5851d9 100644 --- a/tests/test_landscape_sweep_harness.py +++ b/tests/test_landscape_sweep_harness.py @@ -42,7 +42,7 @@ # changes every later run's prompts_sha256; update PROMPTS_SHA256_CURRENT with it. # 2026-09-27: the maintenance rule is derived from the OpenSSF Scorecard Maintained check, and licenses are information only # (never a refutation reason), per the operator's 2026-09-26/27 decisions. -PROMPTS_SHA256_CURRENT = "a9722fec777e5a5d6037be15ccda8ad208a06bba075fcb9832e49b54bcff1fb4" +PROMPTS_SHA256_CURRENT = "f64eec22f82355b18854be9c05c4eba0fdbaf6b6732296fc18c2fae0ff5daf8d" # The 2026-09-26 run's own value, kept in that run's record (evidence/artifacts/landscape-sweep-20260926/README.md); # fixtures below use it as a historical run's recorded prompts_sha256. PROMPTS_SHA256_20260926 = "3adfbed7a83e85da3fd7951032e1fa3a579101772a47b211580065c6b42618d4" @@ -265,6 +265,9 @@ def test_templates_never_refute_on_license_and_name_the_maintenance_rule(self): self.assertIn("not from pushed_at", common) self.assertIn("TOO NEW TO ASSESS (<90 days)", common) self.assertIn("it is stale under the selection principles' maintenance rule", fit) + # A lane that cannot reach the commit evidence (the web-only GPT-6 lane) never refutes on it. + self.assertIn("never excludes or refutes a repository on unknown maintenance", common) + self.assertIn("unknown maintenance is never a reason to refute", fit) def test_only_per_call_placeholders_remain_after_filling(self): frozen = filled_templates() diff --git a/tools/sota-convergence/landscape-sweep/templates.json b/tools/sota-convergence/landscape-sweep/templates.json index ec9e5cd2f..578322fe0 100644 --- a/tools/sota-convergence/landscape-sweep/templates.json +++ b/tools/sota-convergence/landscape-sweep/templates.json @@ -1,8 +1,8 @@ { - "common": "Date: <>. You are one worker in a saturation sweep of the public repository seathatflowsinourveins/native-agent-stack: a portable reference stack of native agent tooling (Claude Code, Codex, MCP, memory/RAG, token efficiency, CI/supply chain, observability) whose north star is US-equities research and historical simulation (NautilusTrader 2.0.0rc5 with an IBKR destination and a separate Alpaca adapter; LEAN as comparison engine), followed by independently qualified paper trading. Live trading is out of scope.\nHosts: linux-wsl2-x86_64 GPU workstation (Threadripper PRO 5975WX 48 CPUs, 102 GiB RAM, RTX 4090 24 GB, Ada: FP8 native, NVFP4 not native) and macos-arm64 Macs (M5 Pro, 24 GB and 64 GB).\nSelection principles: native accounts and supported upstream installation; maintained upstream, by a rule derived from the OpenSSF Scorecard Maintained check (https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained), which scores an archived repository lowest, uses a 90-day window and gives its top score to at least one commit per week and partial credit for maintainers' issue activity: a repository is stale, and excluded whatever its stars, when it is archived or has no commit on its default branch in the last 90 days (establish this with gh api repos///commits?since=&per_page=1 or the same https://api.github.com REST URL, not from pushed_at, which counts any branch); this rule excludes only the zero-activity end, so record lower activity as a maintenance fact without excluding; a repository created less than 90 days ago is too new to assess and must say so at the start of its demonstrated_gap (\"TOO NEW TO ASSESS (<90 days): ...\"); an old release tag on an actively committed branch is not stale (pin the commit when a needed fix exists only on the branch); licenses are recorded as information only and are never a reason to exclude, refute or rank down a candidate (the operator's private experimental environment); no paid SaaS or new credentials unless a demonstrated gap requires it; local-first and privacy-filtered; evidence over popularity (stars and install counts are never evidence of fit).\nRules: primary sources only (GitHub API/pages for releases, tags, stars, pushed_at, archived, license, renames; official docs and changelogs; Hugging Face model cards). Cite the URL for every fact. Read-only: never install, clone-and-run, or execute candidate code; never read or print credentials. Canonical repository form: https://github.com// (follow renames to the current owner). Answer only in the required JSON.\nSkills (installed from adoption/skills/manifest.json, the pinned <> skills trial): use the ones that fit your role and record every skill you actually used in skills_used. Discovery: search-first (survey maintained upstream options before judging) and iterative-retrieval (staged, bounded retrieval). Refutation: verification-before-completion (evidence before any verdict), supply-chain-risk-auditor (maintenance, ownership, advisories, archived/renamed upstreams) and fp-check (verify a claim before calling it true or false). Layer-specific where relevant: agentic-actions-auditor (CI and GitHub automation), security-threat-model / codeql / semgrep / sarif-parsing (security and supply chain), property-based-testing (quality evaluation), mcp-builder (MCP surfaces), modern-python (Python tooling), agent-browser (JavaScript-heavy pages you cannot read otherwise). In Claude Code invoke a skill with the Skill tool; in Codex mention it as $skill-name. Do not load skills you do not use.\nMerit, not incumbency: the current winners are provisional. Being the current winner, the retained control (for example ai-memory in durable-memory is recorded as the retained control, not a measured best), popular, or already installed is NOT evidence of quality. Judge every repository, winners included, on current evidence: measured quality on this requirement (benchmarks, retained comparisons), SOTA-ness (recent releases, capability), maintenance and supply-chain health, and platform fit (license is information only). A candidate that is plausibly better than a winner on such evidence has a demonstrated gap. Report in notes any winner whose standing rests on incumbency rather than evidence.", + "common": "Date: <>. You are one worker in a saturation sweep of the public repository seathatflowsinourveins/native-agent-stack: a portable reference stack of native agent tooling (Claude Code, Codex, MCP, memory/RAG, token efficiency, CI/supply chain, observability) whose north star is US-equities research and historical simulation (NautilusTrader 2.0.0rc5 with an IBKR destination and a separate Alpaca adapter; LEAN as comparison engine), followed by independently qualified paper trading. Live trading is out of scope.\nHosts: linux-wsl2-x86_64 GPU workstation (Threadripper PRO 5975WX 48 CPUs, 102 GiB RAM, RTX 4090 24 GB, Ada: FP8 native, NVFP4 not native) and macos-arm64 Macs (M5 Pro, 24 GB and 64 GB).\nSelection principles: native accounts and supported upstream installation; maintained upstream, by a rule derived from the OpenSSF Scorecard Maintained check (https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained), which scores an archived repository lowest, uses a 90-day window and gives its top score to at least one commit per week and partial credit for maintainers' issue activity: a repository is stale, and excluded whatever its stars, when it is archived or has no commit on its default branch in the last 90 days (establish this with gh api repos///commits?since=&per_page=1 or the same https://api.github.com REST URL, not from pushed_at, which counts any branch; a lane that cannot reach either source, such as a web-only lane, records the commit fact as unknown and never excludes or refutes a repository on unknown maintenance, because a missing result is pending, never refuted); this rule excludes only the zero-activity end, so record lower activity as a maintenance fact without excluding; a repository created less than 90 days ago is too new to assess and must say so at the start of its demonstrated_gap (\"TOO NEW TO ASSESS (<90 days): ...\"); an old release tag on an actively committed branch is not stale (pin the commit when a needed fix exists only on the branch); licenses are recorded as information only and are never a reason to exclude, refute or rank down a candidate (the operator's private experimental environment); no paid SaaS or new credentials unless a demonstrated gap requires it; local-first and privacy-filtered; evidence over popularity (stars and install counts are never evidence of fit).\nRules: primary sources only (GitHub API/pages for releases, tags, stars, pushed_at, archived, license, renames; official docs and changelogs; Hugging Face model cards). Cite the URL for every fact. Read-only: never install, clone-and-run, or execute candidate code; never read or print credentials. Canonical repository form: https://github.com// (follow renames to the current owner). Answer only in the required JSON.\nSkills (installed from adoption/skills/manifest.json, the pinned <> skills trial): use the ones that fit your role and record every skill you actually used in skills_used. Discovery: search-first (survey maintained upstream options before judging) and iterative-retrieval (staged, bounded retrieval). Refutation: verification-before-completion (evidence before any verdict), supply-chain-risk-auditor (maintenance, ownership, advisories, archived/renamed upstreams) and fp-check (verify a claim before calling it true or false). Layer-specific where relevant: agentic-actions-auditor (CI and GitHub automation), security-threat-model / codeql / semgrep / sarif-parsing (security and supply chain), property-based-testing (quality evaluation), mcp-builder (MCP surfaces), modern-python (Python tooling), agent-browser (JavaScript-heavy pages you cannot read otherwise). In Claude Code invoke a skill with the Skill tool; in Codex mention it as $skill-name. Do not load skills you do not use.\nMerit, not incumbency: the current winners are provisional. Being the current winner, the retained control (for example ai-memory in durable-memory is recorded as the retained control, not a measured best), popular, or already installed is NOT evidence of quality. Judge every repository, winners included, on current evidence: measured quality on this requirement (benchmarks, retained comparisons), SOTA-ness (recent releases, capability), maintenance and supply-chain health, and platform fit (license is information only). A candidate that is plausibly better than a winner on such evidence has a demonstrated gap. Report in notes any winner whose standing rests on incumbency rather than evidence.", "discover": "ROLE: discovery researcher for ONE layer.\nLayer input (requirement, current winners and pins, alternatives, upstream drift, open gaps, the previous sweep's survivors/refuted, repositories already known, and seeded candidates from other sessions):\n<>\n<>\nTask: find the strongest CURRENT candidates that could beat, replace or usefully complement this layer's winners for THIS requirement on these hosts. Cover: newest releases and models (last 4 weeks especially), strongly maintained community projects, the seeded candidates (assess every seeded item that is a repository: propose it with the right label or say in notes why not), and any winner health change (archived, renamed, relicensed, major release) which you report in notes.\nPropose at most 6 repositories. Label each:\n- targeted_candidate: worth a bounded native trial against the winner now (a concrete gap and a feasible trial on these hosts).\n- keep_but_compare: credible alternative to compare when the requirement or evidence changes.\n- not_adopted: checked and not worth pursuing (give the decisive reason) -- use this for a prominent/seeded repository you rejected so the rejection is recorded.\nFor each: repository; proposed_label; demonstrated_gap (the specific requirement gap versus the current winners, concrete, not generic praise); comparison_that_would_overturn (the measurable comparison that would decide it); evidence (URLs, each with what it shows); upstream_now (latest_release tag, released_at date, stars, pushed_at, archived, license -- from the GitHub API or page; null when unknowable); source (how you found it).\nRecord your call counts in calls (web_search, web_fetch, gh_api). Put winner-health observations and any seeded item you did not propose in notes. Set layer_id to <>.\nResearch budget (hard): at most 12 web searches, 8 page fetches and 40 GitHub API calls for this layer; stop researching when the budget is spent and answer with what you verified.", "facts": "ROLE: facts/identity refuter for ONE layer. Assume every proposal below may contain a false or unsupported fact.\nLayer: <> -- requirement: <>\nProposals (JSON):\n<>\nFor EACH proposal check with primary sources (GitHub API/pages via gh api or fetch; official docs): the repository exists at that canonical URL (or its rename), is not archived or abandoned contrary to the claim, and the license, latest release, release date, stars/activity and every cited evidence URL actually say what the proposal claims. Vote refuted=true if any material factual claim is false, unsupported by its cited source, or unverifiable; refuted=false only when you verified the facts. Default to refuted=true when uncertain. Give confidence 0..1, short reasoning, and the refs (URLs) you checked. One vote per proposal, same repository string. Set role to facts and layer_id to <>.\nVerification budget (hard): at most 8 web searches, 10 page fetches and 30 GitHub API calls in total for all proposals.", - "fit": "ROLE: fit/standing refuter for ONE layer. Assume every proposal below is wrong for this repository and try to break it.\nLayer input:\n<>\nProposals (JSON):\n<>\nFor EACH proposal vote refuted=true if ANY holds: no concrete, evidence-backed gap versus the current winners for THIS requirement (a plausible advantage in measured quality, SOTA-ness, maintenance or fit counts; incumbency does not count for the winner); it duplicates a winner or an already-known alternative without new evidence; it does not support linux x86_64/WSL2 or macOS arm64 where the layer needs it; it is infeasible on the hosts (e.g. needs more than 24 GB VRAM or more RAM than available, or NVFP4-only weights on Ada); it is stale under the selection principles' maintenance rule (archived, or no default-branch commit in the last 90 days); it requires a paid SaaS or new credentials without a gap that justifies it; it conflicts with the selection principles; or its label is wrong (a targeted_candidate with no feasible bounded trial, or a not_adopted whose rejection reason is itself wrong). Vote refuted=false only when the proposal and its label hold up. Default to refuted=true when uncertain. Give confidence 0..1, reasoning, refs. One vote per proposal, same repository string. Set role to fit and layer_id to <>.\nVerification budget (hard): at most 8 web searches, 10 page fetches and 30 GitHub API calls in total for all proposals.", + "fit": "ROLE: fit/standing refuter for ONE layer. Assume every proposal below is wrong for this repository and try to break it.\nLayer input:\n<>\nProposals (JSON):\n<>\nFor EACH proposal vote refuted=true if ANY holds: no concrete, evidence-backed gap versus the current winners for THIS requirement (a plausible advantage in measured quality, SOTA-ness, maintenance or fit counts; incumbency does not count for the winner); it duplicates a winner or an already-known alternative without new evidence; it does not support linux x86_64/WSL2 or macOS arm64 where the layer needs it; it is infeasible on the hosts (e.g. needs more than 24 GB VRAM or more RAM than available, or NVFP4-only weights on Ada); it is stale under the selection principles' maintenance rule (archived, or no default-branch commit in the last 90 days, established from evidence; unknown maintenance is never a reason to refute); it requires a paid SaaS or new credentials without a gap that justifies it; it conflicts with the selection principles; or its label is wrong (a targeted_candidate with no feasible bounded trial, or a not_adopted whose rejection reason is itself wrong). Vote refuted=false only when the proposal and its label hold up. Default to refuted=true when uncertain. Give confidence 0..1, reasoning, refs. One vote per proposal, same repository string. Set role to fit and layer_id to <>.\nVerification budget (hard): at most 8 web searches, 10 page fetches and 30 GitHub API calls in total for all proposals.", "critic": "ROLE: completeness critic for a <>-layer saturation sweep. Below is a compact summary of every layer: requirement, winners, what was proposed and what survived both refuters.\n<>\nFind at most 8 layers where discovery most likely MISSED a strong current candidate class (a newest release or model from the last 4 weeks, a whole category of tools, a platform-specific option for linux-wsl2 RTX 4090 or macOS arm64, a north-star research or simulation runtime, a security or supply-chain option). For each give the layer_id, the reason, and 2-5 concrete search directions. Also list general observations (e.g. systematic bias). Do not re-propose candidates already proposed.", "followup": "FOLLOW-UP ROUND. The completeness critic flagged this layer: <>\nSearch directions: <>\nAlready proposed in the first round (do not re-propose): <>" } From 5b434740d843ac358d0bb4b6c3d27d8b04301359 Mon Sep 17 00:00:00 2001 From: Scout Date: Sun, 27 Sep 2026 02:12:54 -0400 Subject: [PATCH 5/8] Landscape-sweep GPT-6 lane through OmniRoute: a lane-local Codex home with the token stack at max effort The operator's 2026-09-27 direction: route the sweep's GPT-6 lane through OmniRoute, with the full token-efficiency practice, then run the 20-layer sweep through it. build_args.py --gpt6-provider omniroute --codex-host stages /codex-home: - config.toml: an OmniRoute provider block (cx/gpt-6-astra, effort max, loopback /v1, env_key OMNIROUTE_API_KEY, wire_api responses, no websockets), plus the [mcp_servers.*] tables of codex.config.template.toml rendered for the host by the checkout's render_config.py (serena, ai-memory, socraticode, headroom, codebase-memory, qmd, context-mode). Project and hook trust are left out. - stack-worker.config.toml: the Codex worker profile, copied verbatim. codex_job.py runs that lane with CODEX_HOME set to the staged home, without --ignore-user-config, and with -p stack-worker. The key comes from the OMNIROUTE_API_KEY variable. A keyless loopback gateway gets the "local-loopback" placeholder, and a real key wins. The quota gate is refused for a gateway lane, and inputs.json records the provider. Model names may carry a provider prefix. The native default is unchanged. Tests: 8 new, covering rendering, the runner's argv and env, key handling and refusals. 161 run, OK (3 skipped). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012qwXNtFyUhY5x2dtrm7tG5 --- manifests/evidence.json | 16 +- tests/test_landscape_sweep_harness.py | 156 +++++++++++++++++- .../landscape-sweep/README.md | 29 ++++ .../landscape-sweep/build_args.py | 150 ++++++++++++++++- .../landscape-sweep/codex_job.py | 99 +++++++++-- 5 files changed, 423 insertions(+), 27 deletions(-) diff --git a/manifests/evidence.json b/manifests/evidence.json index 8bd953193..e517d0bac 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -38478,8 +38478,8 @@ }, { "path": "tests/test_landscape_sweep_harness.py", - "sha256": "7f9c7ab0e3555563f1c26a651005cb3affad2ab4f19628bcbdf581fb800e7c17", - "bytes": 148389 + "sha256": "0141d5d0ee5cce9458cd5aaec8ee81e8a4cbe49a55e1bc18f621e16c9ee208b5", + "bytes": 157300 }, { "path": "tests/test_lane_packets.py", @@ -38938,13 +38938,13 @@ }, { "path": "tools/sota-convergence/landscape-sweep/README.md", - "sha256": "3e059eb203d373f36e0fa035e372cf494b40d94cfe786caf2eaa2863065546f8", - "bytes": 34645 + "sha256": "3a88d9b85b226d5480bae6605c1a73fce13a4fbab3c3f65972cf556b5068a9c9", + "bytes": 37492 }, { "path": "tools/sota-convergence/landscape-sweep/build_args.py", - "sha256": "c9d68dfc93e59a994d41e0c9df717fd2fc5e8f934ef7c3da5719b700e98f3f4a", - "bytes": 17188 + "sha256": "b6bf5599b57651728ab67f25636f1483df21d3f84c16c21fd99ac246dc57c0cf", + "bytes": 26317 }, { "path": "tools/sota-convergence/landscape-sweep/build_inputs.py", @@ -38958,8 +38958,8 @@ }, { "path": "tools/sota-convergence/landscape-sweep/codex_job.py", - "sha256": "0a3603ad32aaa4f930beabfe088ff295e3ff31cfecd25adbcc5e542322867e99", - "bytes": 30208 + "sha256": "187551b85c49a5c9c8f62f13ebb9e337bc6c8751fb2b318726a7c00dcde249ea", + "bytes": 35750 }, { "path": "tools/sota-convergence/landscape-sweep/convert.py", diff --git a/tests/test_landscape_sweep_harness.py b/tests/test_landscape_sweep_harness.py index 3bd5851d9..f8b2fca13 100644 --- a/tests/test_landscape_sweep_harness.py +++ b/tests/test_landscape_sweep_harness.py @@ -614,7 +614,8 @@ def test_refusals(self): config = json.load(open(os.environ["FAKE_CODEX_CONFIG"])) stdin, null = os.fstat(0), os.stat(os.devnull) record = {{"argv": sys.argv[1:], "cwd": os.getcwd(), "stdin_devnull": (stdin.st_ino, stdin.st_dev) == (null.st_ino, null.st_dev), - "stdin_read": sys.stdin.read(), "rust_log": os.environ.get("RUST_LOG")}} + "stdin_read": sys.stdin.read(), "rust_log": os.environ.get("RUST_LOG"), + "codex_home": os.environ.get("CODEX_HOME"), "api_key_present": bool(os.environ.get("OMNIROUTE_API_KEY"))}} if config.get("stubborn"): import subprocess subprocess.Popen([sys.executable, "-c", "import os, signal, sys, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); " @@ -653,6 +654,90 @@ def process_alive(pid: int) -> bool: return bool(state) and not state.startswith("Z") +STACK_WORKER_FIXTURE = """model = "gpt-6-astra" +model_reasoning_effort = "max" +web_search = "live" + +[mcp_servers.context-mode] +disabled_tools = ["ctx_upgrade", "ctx_purge"] +""" +TOKEN_MCP_SERVERS = ("serena", "ai-memory", "socraticode", "headroom", "codebase-memory", "qmd", "context-mode") + + +class OmniRouteLaneBuildTests(unittest.TestCase): + """--gpt6-provider omniroute stages a lane-local CODEX_HOME: the provider block, the rendered token MCP servers + and the stack-worker profile, and nothing of the host's interactive trust state.""" + + def stage_lane(self, *extra): + work = stage_work(self) + profile = temp_dir(self) / "stack-worker.config.toml" + profile.write_text(STACK_WORKER_FIXTURE, encoding="utf-8") + return work, profile, build(work, "--gpt6-provider", "omniroute", "--codex-host", "example", + "--stack-worker-profile", profile, *extra) + + def test_lane_home_carries_provider_mcp_servers_and_profile(self): + work, profile, done = self.stage_lane() + self.assertEqual(done.returncode, 0, done.stderr) + config = (work / "codex-home" / "config.toml").read_text(encoding="utf-8") + for line in ('model = "cx/gpt-6-astra"', 'model_provider = "omniroute"', 'model_reasoning_effort = "max"', + "[model_providers.omniroute]", 'base_url = "http://127.0.0.1:20128/v1"', + 'env_key = "OMNIROUTE_API_KEY"', "requires_openai_auth = false", 'wire_api = "responses"'): + self.assertIn(line, config) + for server in TOKEN_MCP_SERVERS: + self.assertIn(f"[mcp_servers.{server}]", config) + self.assertNotIn("supports_websockets = true", config) + self.assertNotIn("[projects.", config) + self.assertNotIn("[hooks.state", config) + self.assertNotIn("${", config) # every template placeholder rendered + self.assertEqual((work / "codex-home" / "stack-worker.config.toml").read_bytes(), profile.read_bytes()) + try: + import tomllib + except ImportError: + tomllib = None + if tomllib is not None: + parsed = tomllib.loads(config) + self.assertEqual(parsed["model_providers"]["omniroute"]["env_key"], "OMNIROUTE_API_KEY") + self.assertEqual(sorted(parsed["mcp_servers"]), sorted(TOKEN_MCP_SERVERS)) + codex = json.loads((work / "staged.json").read_text())["codex"] + self.assertEqual((codex["provider"], codex["codex_home"], codex["profile"], codex["api_key_env"], + codex["model"], codex["effort"]), + ("omniroute", "codex-home", "stack-worker", "OMNIROUTE_API_KEY", "cx/gpt-6-astra", "max")) + self.assertEqual(sorted(codex["lane_home"]["mcp_servers"]), sorted(TOKEN_MCP_SERVERS)) + self.assertNotIn("quota_stop_percent", codex) + self.assertEqual(codex["api_key_placeholder"], "local-loopback") # keyless loopback gateway by default + self.assertNotIn("local-loopback", config) # the placeholder is runtime-only, never in the lane config + # The staged runner reads the same settings back. + lane = codex_job.settings(work) + self.assertEqual((lane["provider"], lane["codex_home"], lane["profile"]), + ("omniroute", work / "codex-home", "stack-worker")) + + def test_lane_refuses_quota_gate_missing_host_and_remote_gateways(self): + for extra, needle in ((("--quota-stop-percent", "90"), "--quota-stop-percent"), + (("--omniroute-base-url", "http://10.0.0.5:20128/v1"), "loopback")): + _, _, done = self.stage_lane(*extra) + self.assertEqual(done.returncode, 2, done.stdout) + self.assertIn(needle, done.stderr) + work = stage_work(self) + done = build(work, "--gpt6-provider", "omniroute") + self.assertEqual(done.returncode, 2, done.stdout) + self.assertIn("--codex-host", done.stderr) + + def test_require_key_stages_no_placeholder(self): + work, _, done = self.stage_lane("--omniroute-require-key") + self.assertEqual(done.returncode, 0, done.stderr) + self.assertNotIn("api_key_placeholder", json.loads((work / "staged.json").read_text())["codex"]) + + def test_native_default_is_unchanged(self): + work = stage_work(self) + done = build(work) + self.assertEqual(done.returncode, 0, done.stderr) + codex = json.loads((work / "staged.json").read_text())["codex"] + self.assertNotIn("provider", codex) + self.assertEqual(codex["model"], "gpt-6-astra") + self.assertFalse((work / "codex-home").exists()) + self.assertEqual(codex_job.settings(work)["codex_home"], None) + + class RunnerCase(unittest.TestCase): def setUp(self): self.work = temp_dir(self) @@ -698,6 +783,75 @@ def record(self): "reasoning_output_tokens": 3}} +class OmniRouteLaneRunnerTests(RunnerCase): + """A gateway lane runs Codex with CODEX_HOME set to the staged lane-local home, -p stack-worker and no + --ignore-user-config, and never starts without its key variable.""" + + def lane(self): + home = self.work / "codex-home" + home.mkdir(exist_ok=True) + (home / "config.toml").write_text('model = "cx/gpt-6-astra"\nmodel_provider = "omniroute"\n', encoding="utf-8") + (home / "stack-worker.config.toml").write_text(STACK_WORKER_FIXTURE, encoding="utf-8") + self.settings({"provider": "omniroute", "codex_home": "codex-home", "profile": "stack-worker", + "api_key_env": "OMNIROUTE_API_KEY", "model": "cx/gpt-6-astra"}) + + def test_lane_uses_its_codex_home_and_profile(self): + self.lane() + self.env["OMNIROUTE_API_KEY"] = "fixture-not-a-key" + result = self.job("gpt6-probe", last=LAST, events=[COMPLETED]) + directory = self.work / "gpt6" / "gpt6-probe" + self.assertEqual(self.record()["argv"], [ + "exec", "-p", "stack-worker", "--skip-git-repo-check", "-s", "read-only", + "-m", "cx/gpt-6-astra", "-c", 'model_reasoning_effort="max"', "-c", 'web_search="live"', + "--output-schema", str(directory / "schema.json"), "-o", str(directory / "last.json"), "--json", + "Reply in JSON."]) + self.assertEqual(self.record()["codex_home"], str(self.work / "codex-home")) + self.assertTrue(self.record()["api_key_present"]) + self.assertEqual((result["status"], result["exit"], result["model"]), ("done", 0, "cx/gpt-6-astra")) + self.assertEqual(json.loads((directory / "inputs.json").read_text())["provider"], "omniroute") + + def test_lane_without_its_key_never_starts_codex(self): + self.lane() + self.env.pop("OMNIROUTE_API_KEY", None) + result = self.job("gpt6-probe", last=LAST, events=[COMPLETED]) + self.assertEqual(result["exit"], codex_job.EXIT_NO_KEY) + self.assertIn("OMNIROUTE_API_KEY is not set", result["stderr_tail"]) + self.assertFalse((self.bin / "record.json").exists()) # the fake codex never ran + + def test_keyless_lane_uses_the_placeholder_only_when_the_variable_is_unset(self): + self.lane() + self.settings({"provider": "omniroute", "codex_home": "codex-home", "profile": "stack-worker", + "api_key_env": "OMNIROUTE_API_KEY", "api_key_placeholder": "local-loopback", + "model": "cx/gpt-6-astra"}) + self.env.pop("OMNIROUTE_API_KEY", None) + result = self.job("gpt6-probe", last=LAST, events=[COMPLETED]) + self.assertEqual(result["exit"], 0) + self.assertTrue(self.record()["api_key_present"]) + lane = codex_job.settings(self.work) + saved = os.environ.pop("OMNIROUTE_API_KEY", None) + try: + self.assertEqual(codex_job.codex_env(lane)["OMNIROUTE_API_KEY"], "local-loopback") + os.environ["OMNIROUTE_API_KEY"] = "operator-value" + self.assertEqual(codex_job.codex_env(lane)["OMNIROUTE_API_KEY"], "operator-value") # a real key wins + finally: + os.environ.pop("OMNIROUTE_API_KEY", None) + if saved is not None: + os.environ["OMNIROUTE_API_KEY"] = saved + + def test_lane_settings_refuse_a_quota_gate_and_a_missing_home(self): + self.lane() + self.settings({"provider": "omniroute", "codex_home": "codex-home", "profile": "stack-worker", + "api_key_env": "OMNIROUTE_API_KEY", "quota_stop_percent": 90}) + with self.assertRaises(codex_job.UsageError): + codex_job.settings(self.work) + self.settings({"provider": "omniroute", "codex_home": "missing-home", "api_key_env": "OMNIROUTE_API_KEY"}) + with self.assertRaises(codex_job.UsageError): + codex_job.settings(self.work) + self.settings({"provider": "omniroute", "codex_home": "../outside", "api_key_env": "OMNIROUTE_API_KEY"}) + with self.assertRaises(codex_job.UsageError): + codex_job.settings(self.work) + + class RunnerTests(RunnerCase): def test_codex_command_line_stdin_and_directory(self): result = self.job("gpt6-probe", last=LAST, events=[{"type": "thread.started", "thread_id": "fixture"}, COMPLETED]) diff --git a/tools/sota-convergence/landscape-sweep/README.md b/tools/sota-convergence/landscape-sweep/README.md index 2fda8a200..d7885ea39 100644 --- a/tools/sota-convergence/landscape-sweep/README.md +++ b/tools/sota-convergence/landscape-sweep/README.md @@ -91,6 +91,35 @@ The GPT-6 lanes run with `--ignore-user-config`, so per-skill `enabled = false` `config.toml` do not apply there. Which skills Codex lists inside the lane is untested; `skills_used` records what each worker says it used. +### GPT-6 through OmniRoute (`--gpt6-provider omniroute`) + +With `build_args.py --gpt6-provider omniroute --codex-host `, the GPT-6 lane runs through the local OmniRoute gateway. OmniRoute pools the operator's accounts. The lane gets its own Codex home, so the host's interactive config never applies. + +`build_args.py` writes `/codex-home/` with two files: +- `config.toml`: + - the provider block: `model = "cx/gpt-6-astra"`, `model_provider = "omniroute"`, `model_reasoning_effort = "max"`, and `[model_providers.omniroute]` with a loopback `base_url` ending in `/v1`, `env_key = "OMNIROUTE_API_KEY"`, `requires_openai_auth = false` and `wire_api = "responses"`. The fields follow the [Codex config reference](https://developers.openai.com/codex/config-reference). + - the `[mcp_servers.*]` tables of `adoption/templates/codex.config.template.toml`. The checkout's own `tools/adoption/render_config.py` renders them for `adoption/hosts/.json`: serena, ai-memory, socraticode, headroom, codebase-memory, qmd and context-mode. +- `stack-worker.config.toml`: the Codex worker profile (`--stack-worker-profile`, default `adoption/templates/codex.stack-worker.config.toml`), copied verbatim. + +What the lane does not carry or allow: +- Project and hook trust are left out, because they describe the host's interactive client. +- `supports_websockets` stays unset. OmniRoute forwards the Codex client version only on its HTTP `/v1/responses` path. +- `--omniroute-base-url` must point at loopback. + +How the runner uses it: +- `codex_job.py` sets `CODEX_HOME` to that home, drops `--ignore-user-config` and adds `-p stack-worker`: `codex exec -p stack-worker --skip-git-repo-check -s read-only -m cx/gpt-6-astra -c model_reasoning_effort="max" -c web_search="live" ...`. +- The key comes from `$OMNIROUTE_API_KEY` in the harness's environment. For a keyless loopback gateway, upstream's non-interactive setup with no login or API key, the staged placeholder `local-loopback` fills an unset variable; Codex's `env_key` only needs the variable to exist. `--omniroute-require-key` stages no placeholder, so a job without the variable ends with exit 6 before codex starts. +- `--quota-stop-percent` is refused with this provider: the quota probe reads the native login, not the gateway's pool. +- A job's `inputs.json` records the provider, so a gateway run never reuses a native job's result. + +Before a full run through the gateway, run the lane's parity check on the staged home, and do not claim a gateway result as max quality until it passes. The check covers: +- max effort reaching the upstream model, shown in the gateway's request log and the rollout's `turn_context`; +- the shell tool; +- an MCP call such as `ctx_execute`; +- `--output-schema` output; +- reported usage; +- whether hosted web search passes through for a custom provider. + ## Evidence contract `saturation_ledger.py --check` and `--append` need the following. Each item names the part of the harness that diff --git a/tools/sota-convergence/landscape-sweep/build_args.py b/tools/sota-convergence/landscape-sweep/build_args.py index 617ccd947..72b4cd2e4 100755 --- a/tools/sota-convergence/landscape-sweep/build_args.py +++ b/tools/sota-convergence/landscape-sweep/build_args.py @@ -63,12 +63,117 @@ "how many GitHub stars does it have? Answer only in the required JSON.") ARGS_LINE = re.compile(r"^const A = args$", re.MULTILINE) SWEEP_ID = re.compile(r"[a-z0-9][a-z0-9._-]{0,127}") +# Kept equal to codex_job.MODEL_NAME: an optional provider prefix for gateway routes, e.g. "cx/gpt-6-astra". +MODEL_NAME = re.compile(r"(?:[A-Za-z0-9][A-Za-z0-9._-]{0,31}/)?[A-Za-z0-9][A-Za-z0-9._-]{0,63}") +# GPT-6 lane providers. native: Codex's own login (--ignore-user-config). omniroute: a lane-local CODEX_HOME whose +# config.toml routes Codex through the local OmniRoute gateway (Codex model_providers with env_key; see +# https://developers.openai.com/codex/config-reference) and carries the token-stack MCP servers. +PROVIDERS = ("native", "omniroute") +OMNIROUTE_DEFAULT_URL = "http://127.0.0.1:20128/v1" +OMNIROUTE_DEFAULT_MODEL = "cx/gpt-6-astra" +OMNIROUTE_KEY_ENV = "OMNIROUTE_API_KEY" +# A loopback OmniRoute set up without a login or API key (upstream `omniroute setup --non-interactive`, REQUIRE_API_KEY +# false) accepts any value; Codex's env_key only needs the variable to exist (upstream CODEX-CLI-CONFIGURATION.md, +# "Local unauthenticated OmniRoute"). The runner uses this placeholder only when the variable is unset. +OMNIROUTE_KEYLESS_PLACEHOLDER = "local-loopback" +LOOPBACK_V1_URL = re.compile(r"http://(?:127\.0\.0\.1|localhost)(?::[0-9]{1,5})?/v1") +LANE_HOME = "codex-home" +LANE_PROFILE = "stack-worker" +CODEX_TEMPLATE = Path("adoption/templates/codex.config.template.toml") +STACK_WORKER_PROFILE = Path("adoption/templates/codex.stack-worker.config.toml") +TABLE_HEADER = re.compile(r"^\s*\[") +MCP_TABLE_HEADER = re.compile(r"^\s*\[mcp_servers\.([A-Za-z0-9_-]+)(?:\.[^\]]+)?\]\s*(?:#.*)?$") def load_templates() -> dict: return load_json(HERE / "templates.json") +def mcp_sections(rendered: str) -> tuple[str, list[str]]: + """The [mcp_servers.*] tables of a rendered Codex config.toml, verbatim and in order, and the server names. + + Top-level keys, [projects.*] trust and [hooks.state.*] trust are left out on purpose: they describe the host's + interactive client, not the lane.""" + kept, names, keep = [], [], False + for line in rendered.splitlines(): + if TABLE_HEADER.match(line): + match = MCP_TABLE_HEADER.match(line) + keep = match is not None + if keep and match.group(1) not in names: + names.append(match.group(1)) + if keep: + kept.append(line) + return "\n".join(kept).strip() + "\n", names + + +def render_codex_template(repo_root: Path, host: str) -> str: + """adoption/templates/codex.config.template.toml rendered for adoption/hosts/.json by the checkout's own + tools/adoption/render_config.py (its load_host_values and render_one), never by a copy of its logic.""" + sys.path.insert(0, str(repo_root / "tools" / "adoption")) + try: + import render_config # noqa: E402 (the checkout's renderer) + finally: + sys.path.pop(0) + try: + return render_config.render_one(repo_root / CODEX_TEMPLATE, render_config.load_host_values(host)) + except render_config.RenderError as error: + raise ValueError(f"rendering {CODEX_TEMPLATE} for host {host!r}: {error}") from None + + +def stage_lane_home(work: Path, *, model: str, base_url: str, host: str, profile: Path, repo_root: Path, + require_key: bool = False) -> dict: + """Write /codex-home: config.toml (the OmniRoute provider block plus the rendered token MCP servers) and + stack-worker.config.toml (the worker profile, verbatim). Return what staged.json records about it.""" + rendered = render_codex_template(repo_root, host) + servers_text, servers = mcp_sections(rendered) + if not servers: + raise ValueError(f"{CODEX_TEMPLATE} rendered for {host!r} has no [mcp_servers.*] table") + profile_bytes = Path(profile).read_bytes() + template_sha = sha256_bytes((repo_root / CODEX_TEMPLATE).read_bytes()) + config = "\n".join([ + "# Lane-local Codex home for the landscape sweep's GPT-6 lane, written by build_args.py", + "# (--gpt6-provider omniroute). It is the lane's whole configuration: Codex runs with CODEX_HOME set here,", + "# without --ignore-user-config, and with -p stack-worker. Project and hook trust are deliberately absent.", + f"# MCP servers: the [mcp_servers.*] tables of {CODEX_TEMPLATE} (sha256 {template_sha}) rendered for host", + f"# {host!r} by tools/adoption/render_config.py. The provider key comes from ${OMNIROUTE_KEY_ENV}; it is never", + "# stored here. supports_websockets stays unset (false): OmniRoute's WebSocket bridge drops the client headers", + "# that carry the Codex version, which only its HTTP /v1/responses path forwards.", + f'model = "{model}"', + 'model_provider = "omniroute"', + 'model_reasoning_effort = "max"', + "", + "[model_providers.omniroute]", + 'name = "OmniRoute"', + f'base_url = "{base_url}"', + f'env_key = "{OMNIROUTE_KEY_ENV}"', + "requires_openai_auth = false", + 'wire_api = "responses"', + "", + servers_text, + ]) + home = work / LANE_HOME + home.mkdir(exist_ok=True) + (home / "config.toml").write_text(config, encoding="utf-8") + (home / f"{LANE_PROFILE}.config.toml").write_bytes(profile_bytes) + try: + import tomllib # Python 3.11+; older interpreters skip this syntax check + except ImportError: + tomllib = None + if tomllib is not None: + for name in ("config.toml", f"{LANE_PROFILE}.config.toml"): + try: + tomllib.loads((home / name).read_text(encoding="utf-8")) + except tomllib.TOMLDecodeError as error: + raise ValueError(f"{LANE_HOME}/{name} is not valid TOML: {error}") from None + keyless = {} if require_key else {"api_key_placeholder": OMNIROUTE_KEYLESS_PLACEHOLDER} + return {"provider": "omniroute", "codex_home": LANE_HOME, "profile": LANE_PROFILE, + "api_key_env": OMNIROUTE_KEY_ENV, **keyless, "base_url": base_url, + "lane_home": {"host": host, "template": str(CODEX_TEMPLATE), "template_sha256": template_sha, + "mcp_servers": servers, + "config_sha256": sha256_bytes((home / "config.toml").read_bytes()), + "profile_sha256": sha256_bytes(profile_bytes)}} + + def fill_build(templates: dict, run_date: str, layer_count: int, skills_checked_at: str) -> dict: """The run's frozen templates; only the per-call placeholders sweep.js and make_prompt.py fill may remain.""" values = {"DATE": run_date, "LAYER_COUNT": str(layer_count), "SKILLS_CHECKED_AT": skills_checked_at} @@ -147,7 +252,7 @@ def git(*args): def stage(work: Path, *, sweep_id: str, run_date: str, selected: list, test: bool, stars, gpt6_model: str, slots: int, lock_dir, skills_checked_at: str | None, embed_script: bool, force: bool, - repo_root: Path = REPO_ROOT, quota_stop_percent: float | None = None) -> dict: + repo_root: Path = REPO_ROOT, quota_stop_percent: float | None = None, lane: dict | None = None) -> dict: jobs = [path.name for path in (work / "gpt6").glob("*") if path.is_dir()] if (work / "gpt6").is_dir() else [] if jobs and not force: raise ValueError(f"{work}/gpt6 already holds {len(jobs)} job(s) from an earlier run; move gpt6/ and prompts/ " @@ -192,6 +297,10 @@ def stage(work: Path, *, sweep_id: str, run_date: str, selected: list, test: boo codex["lock_dir"] = str(Path(lock_dir).expanduser().resolve()) if quota_stop_percent is not None: codex["quota_stop_percent"] = quota_stop_percent + if lane is not None: + codex.update(stage_lane_home(work, model=gpt6_model, base_url=lane["base_url"], host=lane["host"], + profile=lane["profile"], repo_root=repo_root, + require_key=lane.get("require_key", False))) harness_files = sorted([*RUNTIME, "sweep.js", "templates.json", *(f"schemas/{name}.json" for name in SCHEMAS)]) staged = {"schema_version": 1, "kind": "landscape_sweep_staging", "sweep_id": sweep_id, "date": run_date, "test": test, "layers": [layer["layer_id"] for layer in selected], "prompts_sha256": digest, @@ -226,7 +335,22 @@ def main(argv=None) -> int: stars = parser.add_mutually_exclusive_group() stars.add_argument("--stars", type=Path, help="star-candidates.json (default /work/star-candidates.json)") stars.add_argument("--no-stars", action="store_true") - parser.add_argument("--gpt6-model", default="gpt-6-astra") + parser.add_argument("--gpt6-model", default=None, + help=f"default gpt-6-astra; {OMNIROUTE_DEFAULT_MODEL} with --gpt6-provider omniroute") + parser.add_argument("--gpt6-provider", choices=PROVIDERS, default="native", + help="native: Codex's own login (default). omniroute: a lane-local CODEX_HOME that routes " + f"Codex through the local OmniRoute gateway with the token-stack MCP servers; the key " + f"comes from ${OMNIROUTE_KEY_ENV} in the harness's environment") + parser.add_argument("--omniroute-base-url", default=OMNIROUTE_DEFAULT_URL, + help=f"loopback OmniRoute Responses endpoint (default {OMNIROUTE_DEFAULT_URL})") + parser.add_argument("--codex-host", metavar="HOST", + help="adoption/hosts/HOST.json, whose values render the lane's token MCP servers " + "(required with --gpt6-provider omniroute)") + parser.add_argument("--stack-worker-profile", type=Path, + help=f"the Codex worker profile overlay (default {STACK_WORKER_PROFILE} in the checkout)") + parser.add_argument("--omniroute-require-key", action="store_true", + help=f"the gateway requires an API key: stage no {OMNIROUTE_KEYLESS_PLACEHOLDER!r} placeholder, " + f"so a job without ${OMNIROUTE_KEY_ENV} ends before codex starts") parser.add_argument("--slots", type=int, default=3, help="concurrent codex jobs per lock dir (default 3)") parser.add_argument("--lock-dir", help="semaphore directory; share one across sweeps that run at the same time") parser.add_argument("--quota-stop-percent", type=float, metavar="PERCENT", @@ -246,8 +370,24 @@ def main(argv=None) -> int: raise ValueError("--slots must be at least 1") if args.quota_stop_percent is not None and not 0 < args.quota_stop_percent <= 100: raise ValueError("--quota-stop-percent must be above 0 and at most 100") - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}", args.gpt6_model): + gpt6_model = args.gpt6_model or (OMNIROUTE_DEFAULT_MODEL if args.gpt6_provider == "omniroute" + else "gpt-6-astra") + if not MODEL_NAME.fullmatch(gpt6_model): raise ValueError("--gpt6-model is not a model name") + lane = None + if args.gpt6_provider == "omniroute": + if args.quota_stop_percent is not None: + raise ValueError("--quota-stop-percent reads the native Codex login; it cannot gate the OmniRoute " + "account pool, so drop it with --gpt6-provider omniroute") + if not args.codex_host or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}", args.codex_host): + raise ValueError("--gpt6-provider omniroute needs --codex-host HOST (adoption/hosts/HOST.json)") + if not LOOPBACK_V1_URL.fullmatch(args.omniroute_base_url): + raise ValueError("--omniroute-base-url must be a loopback http URL ending in /v1") + profile = args.stack_worker_profile or (args.repo_root.resolve() / STACK_WORKER_PROFILE) + if not Path(profile).is_file(): + raise ValueError(f"stack-worker profile {profile} does not exist; pass --stack-worker-profile") + lane = {"host": args.codex_host, "base_url": args.omniroute_base_url, "profile": Path(profile).resolve(), + "require_key": args.omniroute_require_key} if args.skills_checked_at: date.fromisoformat(args.skills_checked_at) layers = load_json(work / "layers.json") @@ -260,9 +400,9 @@ def main(argv=None) -> int: if stars_path is not None and not Path(stars_path).is_file(): raise ValueError(f"--stars {stars_path} does not exist") summary = stage(work, sweep_id=args.sweep_id, run_date=args.date, selected=selected, test=bool(args.smoke), - stars=stars_path, gpt6_model=args.gpt6_model, slots=args.slots, lock_dir=args.lock_dir, + stars=stars_path, gpt6_model=gpt6_model, slots=args.slots, lock_dir=args.lock_dir, skills_checked_at=args.skills_checked_at, embed_script=not args.no_embed, force=args.force, - repo_root=args.repo_root.resolve(), quota_stop_percent=args.quota_stop_percent) + repo_root=args.repo_root.resolve(), quota_stop_percent=args.quota_stop_percent, lane=lane) except (ValueError, OSError, KeyError) as error: print(f"build_args.py: {error}", file=sys.stderr) return 2 diff --git a/tools/sota-convergence/landscape-sweep/codex_job.py b/tools/sota-convergence/landscape-sweep/codex_job.py index 0bc02440e..c0cf1e2e2 100755 --- a/tools/sota-convergence/landscape-sweep/codex_job.py +++ b/tools/sota-convergence/landscape-sweep/codex_job.py @@ -15,6 +15,15 @@ codex exec --ignore-user-config --skip-git-repo-check -s read-only -m gpt-6-astra \ -c model_reasoning_effort="max" -c web_search="live" --output-schema /schema.json -o /last.json --json "" /codex-home instead, drops --ignore-user-config (that home IS the +configuration: the OmniRoute provider block plus the token MCP servers) and adds `-p ` (the stack-worker +profile layered over it), so the command becomes `codex exec -p stack-worker --skip-git-repo-check -s read-only +-m cx/gpt-6-astra ...`. The provider key comes from the variable codex.api_key_env (OMNIROUTE_API_KEY) in the +operator's environment. For a keyless loopback gateway, codex.api_key_placeholder ("local-loopback") fills an +unset variable, because Codex's env_key only needs it to exist. Without either, a job ends with exit 6 before codex +starts. The quota gate reads the native login only, so it is refused for a gateway lane. + --ignore-user-config keeps the host's Codex config out of the lane, the sandbox is read-only, stdin is /dev/null (background `codex exec` otherwise waits on stdin), and the effort is max. Never ultra: ultra lets Codex delegate to sub-agents, which breaks the one-model lane. Codex runs without the caller's RUST_LOG, so its stderr carries only @@ -83,11 +92,16 @@ # The prompt is one argv string, as in the 2026-09-26 runner; Linux caps one argument at 131072 bytes. MAX_PROMPT_BYTES = 120_000 JOB_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}") -MODEL_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}") +# An optional provider prefix is allowed for gateway routes, e.g. OmniRoute's "cx/gpt-6-astra". +MODEL_NAME = re.compile(r"(?:[A-Za-z0-9][A-Za-z0-9._-]{0,31}/)?[A-Za-z0-9][A-Za-z0-9._-]{0,63}") +PROVIDERS = ("native", "omniroute") +ENV_NAME = re.compile(r"[A-Z_][A-Z0-9_]{0,63}") +PROFILE_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}") # Everything one attempt writes; an earlier attempt's files move together to attempts//. ATTEMPT_FILES = ("events.jsonl", "stderr.txt", "last.json", "started", "finished", "exit", "slot", "model", "codex_version", "done", "prompt.txt", "schema.json", "inputs.json", "runner.log", "quota.json") EXIT_LIMIT, EXIT_TIMEOUT, EXIT_NO_CODEX, EXIT_REFUSED = 3, 124, 127, 2 +EXIT_NO_KEY = 6 # a gateway lane whose API key variable is unset in the runner's environment class UsageError(Exception): @@ -144,9 +158,45 @@ def settings(base: Path) -> dict: out["quota_stop_percent"] = None if stop is None else float(stop) if not 0 < out["quota_timeout_s"] <= 600: raise UsageError("codex.quota_timeout_s must be above 0 and at most 600") + out.update(lane_settings(base, staged)) + if out["codex_home"] is not None and out["quota_stop_percent"] is not None: + raise UsageError("codex.quota_stop_percent reads the native Codex login's usage; it cannot gate a gateway " + "provider's account pool, so it must be absent when codex.provider is not native") return out +def lane_settings(base: Path, staged: dict) -> dict: + """The GPT-6 lane's provider. native (the default) runs Codex with --ignore-user-config against the caller's + login. A gateway provider (omniroute) runs Codex with CODEX_HOME set to the staged lane-local home + /: its config.toml (the provider block and the token MCP servers) and its profile file are + the lane's whole configuration, and the provider's API key comes from the named environment variable.""" + provider = str(staged.get("provider") or "native") + if provider not in PROVIDERS: + raise UsageError(f"codex.provider {provider!r} must be one of {', '.join(PROVIDERS)}") + lane = {"provider": provider, "codex_home": None, "profile": None, "api_key_env": None, + "api_key_placeholder": None} + if provider == "native": + return lane + home = staged.get("codex_home") + if not isinstance(home, str) or not home or Path(home).is_absolute() or ".." in Path(home).parts: + raise UsageError("codex.codex_home must name a directory inside the work directory") + if not (base / home / "config.toml").is_file(): + raise UsageError(f"codex.codex_home {home!r} holds no config.toml; restage with build_args.py") + profile = staged.get("profile") + if profile is not None and not (isinstance(profile, str) and PROFILE_NAME.fullmatch(profile)): + raise UsageError(f"codex.profile {profile!r} is not a profile name") + if profile is not None and not (base / home / f"{profile}.config.toml").is_file(): + raise UsageError(f"codex.profile {profile!r} has no {profile}.config.toml in the lane home; restage") + key = staged.get("api_key_env") + if not isinstance(key, str) or not ENV_NAME.fullmatch(key): + raise UsageError("codex.api_key_env must name the environment variable that holds the provider key") + placeholder = staged.get("api_key_placeholder") + if placeholder is not None and not (isinstance(placeholder, str) and PROFILE_NAME.fullmatch(placeholder)): + raise UsageError("codex.api_key_placeholder must be a short plain token (a keyless loopback gateway's value)") + lane.update(codex_home=base / home, profile=profile, api_key_env=key, api_key_placeholder=placeholder) + return lane + + def job_dir(base: Path, job: str) -> Path: if not JOB_ID.fullmatch(job or ""): raise UsageError(f"job id {job!r} must match {JOB_ID.pattern}") @@ -251,18 +301,28 @@ def limit_error(directory: Path) -> bool: for line in (read(directory / "stderr.txt") or "").splitlines()) -def codex_env() -> dict: - """The caller's environment without Rust tracing settings (see the module docstring).""" - return {key: value for key, value in os.environ.items() if not key.startswith("RUST_LOG")} +def codex_env(lane: dict | None = None) -> dict: + """The caller's environment without Rust tracing settings (see the module docstring). For a gateway lane, + CODEX_HOME points at the staged lane-local home, never at the caller's; the key variable passes through as it is + and is never written anywhere.""" + env = {key: value for key, value in os.environ.items() if not key.startswith("RUST_LOG")} + if lane is not None and lane.get("codex_home") is not None: + env["CODEX_HOME"] = str(lane["codex_home"]) + if not env.get(lane["api_key_env"]) and lane.get("api_key_placeholder"): + env[lane["api_key_env"]] = lane["api_key_placeholder"] # keyless loopback gateway; a real key wins + return env def sha256_hex(data: bytes) -> str: return hashlib.sha256(data).hexdigest() -def job_inputs(prompt: bytes, schema: bytes, model: str) -> dict: - return {"prompt_sha256": sha256_hex(prompt), "schema_sha256": sha256_hex(schema), "model": model, - "effort": EFFORT} +def job_inputs(prompt: bytes, schema: bytes, model: str, provider: str = "native") -> dict: + inputs = {"prompt_sha256": sha256_hex(prompt), "schema_sha256": sha256_hex(schema), "model": model, + "effort": EFFORT} + if provider != "native": # a gateway run never reuses a native job's result, or the other way round + inputs["provider"] = provider + return inputs def read_json(path: Path): @@ -288,10 +348,16 @@ def archive_attempt(directory: Path) -> int | None: return number -def codex_argv(codex: str, directory: Path, model: str, prompt: str) -> list[str]: +def codex_argv(codex: str, directory: Path, model: str, prompt: str, lane: dict | None = None) -> list[str]: # Live web search: `--search` before exec (and no flag) sends external_web_access false, a cached index; # only web_search="live" sends true (evidence/artifacts/sota-refresh-20260926/codex/results/websearch-*.json). - return [codex, "exec", "--ignore-user-config", "--skip-git-repo-check", "-s", "read-only", + if lane is not None and lane.get("codex_home") is not None: + # The lane-local CODEX_HOME is the whole configuration, so --ignore-user-config (which drops + # $CODEX_HOME/config.toml) must not be passed; the profile layers .config.toml over it (profile-v2). + head = [codex, "exec", *(["-p", lane["profile"]] if lane.get("profile") else [])] + else: + head = [codex, "exec", "--ignore-user-config"] + return [*head, "--skip-git-repo-check", "-s", "read-only", "-m", model, "-c", f'model_reasoning_effort="{EFFORT}"', "-c", 'web_search="live"', "--output-schema", str(directory / "schema.json"), "-o", str(directory / "last.json"), "--json", prompt] @@ -311,7 +377,8 @@ def start(base: Path, job: str, prompt_file: str, schema_file: str) -> int: directory = job_dir(base, job) prompt_bytes = Path(prompt_file).read_bytes() schema_bytes = Path(schema_file).read_bytes() - inputs = job_inputs(prompt_bytes, schema_bytes, settings(base)["model"]) # also refuses a broken staged.json + config = settings(base) # also refuses a broken staged.json + inputs = job_inputs(prompt_bytes, schema_bytes, config["model"], config["provider"]) changed = False if (directory / "done").exists() and exit_code(directory) == 0: if read_json(directory / "inputs.json") == inputs: @@ -530,6 +597,12 @@ def run(base: Path, job: str) -> int: write_atomic(directory / "stderr.txt", "codex is not on PATH\n") finish(directory, EXIT_NO_CODEX) return EXIT_NO_CODEX + lane = config if config["codex_home"] is not None else None + if lane is not None and not os.environ.get(lane["api_key_env"]) and not lane["api_key_placeholder"]: + write_atomic(directory / "stderr.txt", f"{lane['api_key_env']} is not set in the runner's environment; start " + "the harness with the gateway key loaded from its store by pointer (docs/secret-storage.md)\n") + finish(directory, EXIT_NO_KEY) + return EXIT_NO_KEY version = codex_version(codex) if version: write_atomic(directory / "codex_version", version + "\n") @@ -537,9 +610,9 @@ def run(base: Path, job: str) -> int: prompt = (directory / "prompt.txt").read_text(encoding="utf-8").rstrip("\n") (base / "empty").mkdir(exist_ok=True) with open(directory / "events.jsonl", "wb") as events, open(directory / "stderr.txt", "wb") as errors: - process = subprocess.Popen(codex_argv(codex, directory, config["model"], prompt), cwd=str(base / "empty"), - stdin=subprocess.DEVNULL, stdout=events, stderr=errors, - pass_fds=(slot_fd, lock_fd), start_new_session=True, env=codex_env()) + process = subprocess.Popen(codex_argv(codex, directory, config["model"], prompt, lane), + cwd=str(base / "empty"), stdin=subprocess.DEVNULL, stdout=events, stderr=errors, + pass_fds=(slot_fd, lock_fd), start_new_session=True, env=codex_env(lane)) try: code = process.wait(timeout=config["timeout_s"]) except subprocess.TimeoutExpired: From 4709f17fae564f47ad65b81dfaf707567e21ab0f Mon Sep 17 00:00:00 2001 From: Scout Date: Sun, 27 Sep 2026 02:17:45 -0400 Subject: [PATCH 6/8] OmniRoute lane: --codex-host accepts a private host value file by path Real hosts' value files are gitignored (adoption/hosts/*.json), so they exist only in the checkout that owns them, never in a worktree. --codex-host now takes either a host name (adoption/hosts/.json, through render_config's load_host_values) or a path to a *.json host value file, checked for the same flat string shape. The lane record and the config.toml header carry only the file's stem, never its private location. Staged for this workstation with its private file, the lane home is provider omniroute, cx/gpt-6-astra, effort max, profile stack-worker and the seven token MCP servers, and it is valid TOML. Tests: one new; 162 run, OK (3 skipped). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012qwXNtFyUhY5x2dtrm7tG5 --- manifests/evidence.json | 8 ++-- tests/test_landscape_sweep_harness.py | 18 ++++++++ .../landscape-sweep/build_args.py | 41 ++++++++++++++----- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/manifests/evidence.json b/manifests/evidence.json index e517d0bac..3d88a46da 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -38478,8 +38478,8 @@ }, { "path": "tests/test_landscape_sweep_harness.py", - "sha256": "0141d5d0ee5cce9458cd5aaec8ee81e8a4cbe49a55e1bc18f621e16c9ee208b5", - "bytes": 157300 + "sha256": "aba004d39b42403e97cd4867c1af349dcc522ae58614fe9e296b62b894cd8fc1", + "bytes": 158566 }, { "path": "tests/test_lane_packets.py", @@ -38943,8 +38943,8 @@ }, { "path": "tools/sota-convergence/landscape-sweep/build_args.py", - "sha256": "b6bf5599b57651728ab67f25636f1483df21d3f84c16c21fd99ac246dc57c0cf", - "bytes": 26317 + "sha256": "e67dee42da7881120b03b24317cfd1b085b763626c10a0ec7813106cc4f8904f", + "bytes": 27727 }, { "path": "tools/sota-convergence/landscape-sweep/build_inputs.py", diff --git a/tests/test_landscape_sweep_harness.py b/tests/test_landscape_sweep_harness.py index f8b2fca13..182b07b2e 100644 --- a/tests/test_landscape_sweep_harness.py +++ b/tests/test_landscape_sweep_harness.py @@ -722,6 +722,24 @@ def test_lane_refuses_quota_gate_missing_host_and_remote_gateways(self): self.assertEqual(done.returncode, 2, done.stdout) self.assertIn("--codex-host", done.stderr) + def test_codex_host_accepts_a_private_host_value_file_by_path(self): + private = temp_dir(self) / "privatehost.json" + private.write_text((ROOT / "adoption" / "hosts" / "example.json").read_text(encoding="utf-8"), encoding="utf-8") + work = stage_work(self) + profile = temp_dir(self) / "stack-worker.config.toml" + profile.write_text(STACK_WORKER_FIXTURE, encoding="utf-8") + done = build(work, "--gpt6-provider", "omniroute", "--codex-host", private, "--stack-worker-profile", profile) + self.assertEqual(done.returncode, 0, done.stderr) + config = (work / "codex-home" / "config.toml").read_text(encoding="utf-8") + self.assertIn("[mcp_servers.context-mode]", config) + self.assertNotIn(str(private.parent), config) # the private file's location stays out of the lane record + codex = json.loads((work / "staged.json").read_text())["codex"] + self.assertEqual(codex["lane_home"]["host"], "privatehost") + missing = build(stage_work(self), "--gpt6-provider", "omniroute", "--codex-host", str(private) + ".gone.json", + "--stack-worker-profile", profile) + self.assertEqual(missing.returncode, 2) + self.assertIn("does not exist", missing.stderr) + def test_require_key_stages_no_placeholder(self): work, _, done = self.stage_lane("--omniroute-require-key") self.assertEqual(done.returncode, 0, done.stderr) diff --git a/tools/sota-convergence/landscape-sweep/build_args.py b/tools/sota-convergence/landscape-sweep/build_args.py index 72b4cd2e4..528f2fbb4 100755 --- a/tools/sota-convergence/landscape-sweep/build_args.py +++ b/tools/sota-convergence/landscape-sweep/build_args.py @@ -106,18 +106,30 @@ def mcp_sections(rendered: str) -> tuple[str, list[str]]: return "\n".join(kept).strip() + "\n", names +def host_values_from_file(path: Path) -> dict: + """A private host value file given by path. Real hosts' files are gitignored (adoption/hosts/*.json), so they + exist only in the checkout that owns them, never in a worktree. Same shape render_config requires.""" + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict) or not all(isinstance(value, str) for value in data.values()): + raise ValueError(f"{path} must be a flat JSON object of string values") + return data + + def render_codex_template(repo_root: Path, host: str) -> str: - """adoption/templates/codex.config.template.toml rendered for adoption/hosts/.json by the checkout's own - tools/adoption/render_config.py (its load_host_values and render_one), never by a copy of its logic.""" + """adoption/templates/codex.config.template.toml rendered for a host by the checkout's own + tools/adoption/render_config.py (render_one), never by a copy of its logic. host is a name + (adoption/hosts/.json, via render_config.load_host_values) or a path to a private host value file.""" sys.path.insert(0, str(repo_root / "tools" / "adoption")) try: import render_config # noqa: E402 (the checkout's renderer) finally: sys.path.pop(0) try: - return render_config.render_one(repo_root / CODEX_TEMPLATE, render_config.load_host_values(host)) + values = (host_values_from_file(Path(host)) if host.endswith(".json") + else render_config.load_host_values(host)) + return render_config.render_one(repo_root / CODEX_TEMPLATE, values) except render_config.RenderError as error: - raise ValueError(f"rendering {CODEX_TEMPLATE} for host {host!r}: {error}") from None + raise ValueError(f"rendering {CODEX_TEMPLATE} for host {Path(host).name!r}: {error}") from None def stage_lane_home(work: Path, *, model: str, base_url: str, host: str, profile: Path, repo_root: Path, @@ -125,6 +137,7 @@ def stage_lane_home(work: Path, *, model: str, base_url: str, host: str, profile """Write /codex-home: config.toml (the OmniRoute provider block plus the rendered token MCP servers) and stack-worker.config.toml (the worker profile, verbatim). Return what staged.json records about it.""" rendered = render_codex_template(repo_root, host) + host_label = Path(host).stem if host.endswith(".json") else host # never a private path in the lane record servers_text, servers = mcp_sections(rendered) if not servers: raise ValueError(f"{CODEX_TEMPLATE} rendered for {host!r} has no [mcp_servers.*] table") @@ -135,7 +148,7 @@ def stage_lane_home(work: Path, *, model: str, base_url: str, host: str, profile "# (--gpt6-provider omniroute). It is the lane's whole configuration: Codex runs with CODEX_HOME set here,", "# without --ignore-user-config, and with -p stack-worker. Project and hook trust are deliberately absent.", f"# MCP servers: the [mcp_servers.*] tables of {CODEX_TEMPLATE} (sha256 {template_sha}) rendered for host", - f"# {host!r} by tools/adoption/render_config.py. The provider key comes from ${OMNIROUTE_KEY_ENV}; it is never", + f"# {host_label!r} by tools/adoption/render_config.py. The provider key comes from ${OMNIROUTE_KEY_ENV}; it is never", "# stored here. supports_websockets stays unset (false): OmniRoute's WebSocket bridge drops the client headers", "# that carry the Codex version, which only its HTTP /v1/responses path forwards.", f'model = "{model}"', @@ -168,7 +181,8 @@ def stage_lane_home(work: Path, *, model: str, base_url: str, host: str, profile keyless = {} if require_key else {"api_key_placeholder": OMNIROUTE_KEYLESS_PLACEHOLDER} return {"provider": "omniroute", "codex_home": LANE_HOME, "profile": LANE_PROFILE, "api_key_env": OMNIROUTE_KEY_ENV, **keyless, "base_url": base_url, - "lane_home": {"host": host, "template": str(CODEX_TEMPLATE), "template_sha256": template_sha, + "lane_home": {"host": host_label, + "template": str(CODEX_TEMPLATE), "template_sha256": template_sha, "mcp_servers": servers, "config_sha256": sha256_bytes((home / "config.toml").read_bytes()), "profile_sha256": sha256_bytes(profile_bytes)}} @@ -344,8 +358,9 @@ def main(argv=None) -> int: parser.add_argument("--omniroute-base-url", default=OMNIROUTE_DEFAULT_URL, help=f"loopback OmniRoute Responses endpoint (default {OMNIROUTE_DEFAULT_URL})") parser.add_argument("--codex-host", metavar="HOST", - help="adoption/hosts/HOST.json, whose values render the lane's token MCP servers " - "(required with --gpt6-provider omniroute)") + help="a host name (adoption/hosts/HOST.json in the checkout) or a path to a private host " + "value file (*.json; real hosts' files are gitignored), whose values render the lane's " + "token MCP servers (required with --gpt6-provider omniroute)") parser.add_argument("--stack-worker-profile", type=Path, help=f"the Codex worker profile overlay (default {STACK_WORKER_PROFILE} in the checkout)") parser.add_argument("--omniroute-require-key", action="store_true", @@ -379,8 +394,14 @@ def main(argv=None) -> int: if args.quota_stop_percent is not None: raise ValueError("--quota-stop-percent reads the native Codex login; it cannot gate the OmniRoute " "account pool, so drop it with --gpt6-provider omniroute") - if not args.codex_host or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}", args.codex_host): - raise ValueError("--gpt6-provider omniroute needs --codex-host HOST (adoption/hosts/HOST.json)") + if not args.codex_host: + raise ValueError("--gpt6-provider omniroute needs --codex-host HOST (a name under adoption/hosts/ " + "or a path to a private host value file)") + if args.codex_host.endswith(".json"): + if not Path(args.codex_host).is_file(): + raise ValueError(f"--codex-host file {args.codex_host} does not exist") + elif not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}", args.codex_host): + raise ValueError("--codex-host must be a host name or a path to a *.json host value file") if not LOOPBACK_V1_URL.fullmatch(args.omniroute_base_url): raise ValueError("--omniroute-base-url must be a loopback http URL ending in /v1") profile = args.stack_worker_profile or (args.repo_root.resolve() / STACK_WORKER_PROFILE) From 8780d93e4bc8a35b1a684c7fb583682005244d19 Mon Sep 17 00:00:00 2001 From: Scout Date: Sun, 27 Sep 2026 02:34:22 -0400 Subject: [PATCH 7/8] #387 GPT-6 review repair: no shell snapshot, standalone web search, fail-closed MCP extraction The GPT-6 cross-family review of 5b434740 returned needs_changes: 1 high, 2 medium and 1 low finding. - high: Codex's default shell snapshot wrote the exported environment, the provider key included, into codex-home/shell_snapshots/*.sh at mode 0644 (shell_snapshot_exports.rs at rust-v0.157.1). The lane config now sets [features] shell_snapshot = false. - medium: GPT-6 Astra runs Responses Lite, which carries no hosted tools, and custom providers default to no standalone search. So the lane gets no web search. The provider now advertises supports_standalone_web_search = true (Codex advanced config docs) and [features] standalone_web_search = true (under development in 0.157.1). The parity check must still show that OmniRoute serves the endpoint. - medium: the default worker profile arrives with the Codex worker-lane change, so the error now names --stack-worker-profile. The README states the dependency. - low: the MCP extraction accepts quoted and spaced table headers, and is checked against a real TOML parse. An extraction that parses badly or names different servers than the whole rendered config fails closed. A blank key value counts as unset, since Codex rejects a blank env_key. The README records the isolation limits: a trusted cwd .codex/config.toml and the system config still layer in; the runner's cwd is empty, and this workstation has no /etc/codex/config.toml. Tests: 164 run, OK (3 skipped). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012qwXNtFyUhY5x2dtrm7tG5 --- manifests/evidence.json | 16 ++++---- tests/test_landscape_sweep_harness.py | 30 ++++++++++++++ .../landscape-sweep/README.md | 13 ++++++ .../landscape-sweep/build_args.py | 40 ++++++++++++++++--- .../landscape-sweep/codex_job.py | 4 +- 5 files changed, 88 insertions(+), 15 deletions(-) diff --git a/manifests/evidence.json b/manifests/evidence.json index 3d88a46da..6c465c14b 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -38478,8 +38478,8 @@ }, { "path": "tests/test_landscape_sweep_harness.py", - "sha256": "aba004d39b42403e97cd4867c1af349dcc522ae58614fe9e296b62b894cd8fc1", - "bytes": 158566 + "sha256": "d59fc347a50a17b68f90c786dd51d4843d78867fa55e4c60db35e7eed5a61d61", + "bytes": 160583 }, { "path": "tests/test_lane_packets.py", @@ -38938,13 +38938,13 @@ }, { "path": "tools/sota-convergence/landscape-sweep/README.md", - "sha256": "3a88d9b85b226d5480bae6605c1a73fce13a4fbab3c3f65972cf556b5068a9c9", - "bytes": 37492 + "sha256": "f6019a3a5301bf6bdbe20212b52556601b880f535693b4552a773d2473299af7", + "bytes": 39012 }, { "path": "tools/sota-convergence/landscape-sweep/build_args.py", - "sha256": "e67dee42da7881120b03b24317cfd1b085b763626c10a0ec7813106cc4f8904f", - "bytes": 27727 + "sha256": "10dc350aaf3dc29ce727758cc8bc421480678aab8540884c32afc2e938a8339a", + "bytes": 29780 }, { "path": "tools/sota-convergence/landscape-sweep/build_inputs.py", @@ -38958,8 +38958,8 @@ }, { "path": "tools/sota-convergence/landscape-sweep/codex_job.py", - "sha256": "187551b85c49a5c9c8f62f13ebb9e337bc6c8751fb2b318726a7c00dcde249ea", - "bytes": 35750 + "sha256": "dce6ca4e908a151c4b509a207a15dc24f0d9f022cbe7031cb5eb87b1a9ec0921", + "bytes": 35782 }, { "path": "tools/sota-convergence/landscape-sweep/convert.py", diff --git a/tests/test_landscape_sweep_harness.py b/tests/test_landscape_sweep_harness.py index 182b07b2e..c2f4be3c4 100644 --- a/tests/test_landscape_sweep_harness.py +++ b/tests/test_landscape_sweep_harness.py @@ -686,6 +686,10 @@ def test_lane_home_carries_provider_mcp_servers_and_profile(self): for server in TOKEN_MCP_SERVERS: self.assertIn(f"[mcp_servers.{server}]", config) self.assertNotIn("supports_websockets = true", config) + # The key never reaches a shell snapshot file, and GPT-6 Astra's search is Codex's standalone web.run. + for line in ("[features]", "shell_snapshot = false", "standalone_web_search = true", + "supports_standalone_web_search = true"): + self.assertIn(line, config) self.assertNotIn("[projects.", config) self.assertNotIn("[hooks.state", config) self.assertNotIn("${", config) # every template placeholder rendered @@ -697,6 +701,8 @@ def test_lane_home_carries_provider_mcp_servers_and_profile(self): if tomllib is not None: parsed = tomllib.loads(config) self.assertEqual(parsed["model_providers"]["omniroute"]["env_key"], "OMNIROUTE_API_KEY") + self.assertIs(parsed["model_providers"]["omniroute"]["supports_standalone_web_search"], True) + self.assertEqual(parsed["features"], {"shell_snapshot": False, "standalone_web_search": True}) self.assertEqual(sorted(parsed["mcp_servers"]), sorted(TOKEN_MCP_SERVERS)) codex = json.loads((work / "staged.json").read_text())["codex"] self.assertEqual((codex["provider"], codex["codex_home"], codex["profile"], codex["api_key_env"], @@ -740,6 +746,28 @@ def test_codex_host_accepts_a_private_host_value_file_by_path(self): self.assertEqual(missing.returncode, 2) self.assertIn("does not exist", missing.stderr) + def test_mcp_extraction_accepts_quoted_and_spaced_headers_and_fails_closed(self): + text = ('model = "x"\n[mcp_servers."demo"]\ncommand = "a"\n[ mcp_servers.plain ]\ncommand = "b"\n' + '[mcp_servers.plain.env]\nK = "v"\n[projects."/p"]\ntrust_level = "trusted"\n') + sections, names = build_args.mcp_sections(text) + self.assertEqual(names, ["demo", "plain"]) + self.assertNotIn("[projects.", sections) + try: + import tomllib + except ImportError: + return + self.assertEqual(sorted(tomllib.loads(sections)["mcp_servers"]), ["demo", "plain"]) + # A server the line extraction cannot see (declared with inline-table syntax under [mcp_servers]) fails closed. + with self.assertRaises(ValueError): + build_args.mcp_sections('[mcp_servers]\nhidden = { command = "c" }\n[mcp_servers.plain]\ncommand = "b"\n') + + def test_missing_profile_names_the_flag(self): + work = stage_work(self) + done = build(work, "--gpt6-provider", "omniroute", "--codex-host", "example", + "--stack-worker-profile", temp_dir(self) / "absent.config.toml") + self.assertEqual(done.returncode, 2, done.stdout) + self.assertIn("pass --stack-worker-profile", done.stderr) + def test_require_key_stages_no_placeholder(self): work, _, done = self.stage_lane("--omniroute-require-key") self.assertEqual(done.returncode, 0, done.stderr) @@ -848,6 +876,8 @@ def test_keyless_lane_uses_the_placeholder_only_when_the_variable_is_unset(self) lane = codex_job.settings(self.work) saved = os.environ.pop("OMNIROUTE_API_KEY", None) try: + self.assertEqual(codex_job.codex_env(lane)["OMNIROUTE_API_KEY"], "local-loopback") + os.environ["OMNIROUTE_API_KEY"] = " " # blank counts as unset: Codex rejects a blank env_key value self.assertEqual(codex_job.codex_env(lane)["OMNIROUTE_API_KEY"], "local-loopback") os.environ["OMNIROUTE_API_KEY"] = "operator-value" self.assertEqual(codex_job.codex_env(lane)["OMNIROUTE_API_KEY"], "operator-value") # a real key wins diff --git a/tools/sota-convergence/landscape-sweep/README.md b/tools/sota-convergence/landscape-sweep/README.md index d7885ea39..bd67b6b4e 100644 --- a/tools/sota-convergence/landscape-sweep/README.md +++ b/tools/sota-convergence/landscape-sweep/README.md @@ -101,10 +101,23 @@ With `build_args.py --gpt6-provider omniroute --codex-host `, the GPT-6 la - the `[mcp_servers.*]` tables of `adoption/templates/codex.config.template.toml`. The checkout's own `tools/adoption/render_config.py` renders them for `adoption/hosts/.json`: serena, ai-memory, socraticode, headroom, codebase-memory, qmd and context-mode. - `stack-worker.config.toml`: the Codex worker profile (`--stack-worker-profile`, default `adoption/templates/codex.stack-worker.config.toml`), copied verbatim. +What the lane config also sets: +- **`[features] shell_snapshot = false`.** Codex's shell snapshot writes the exported environment, the provider key included, into `/shell_snapshots/*.sh` with mode 0644 (`codex-rs/shell-command/src/shell_snapshot_exports.rs` at rust-v0.157.1). A GPT-6 probe reproduced this. +- **Web search for GPT-6 Astra.** Astra runs Responses Lite, which carries no hosted tools, so search reaches it only as Codex's standalone web search (`web.run`). The lane therefore sets: + - `supports_standalone_web_search = true` on the provider, which defaults to false for custom providers ([Codex advanced config](https://learn.chatgpt.com/docs/config-file/config-advanced)); + - `[features] standalone_web_search = true`, which is under development in 0.157.1. + + The capability flag alone enables nothing: OmniRoute must serve a compatible endpoint, and the parity check below has to show it working before a sweep counts on GPT-6 search through the gateway. + What the lane does not carry or allow: - Project and hook trust are left out, because they describe the host's interactive client. - `supports_websockets` stays unset. OmniRoute forwards the Codex client version only on its HTTP `/v1/responses` path. - `--omniroute-base-url` must point at loopback. +- The default worker profile, `adoption/templates/codex.stack-worker.config.toml`, arrives with the Codex worker-lane change. Until the checkout has it, pass `--stack-worker-profile PATH`. + +Isolation limits: +- `CODEX_HOME` replaces the user-config location, but a trusted `.codex/config.toml` in the working directory and the system config still layer in. +- The runner works in the empty `/empty`, so no project layer applies there. A host system config (`/etc/codex/config.toml`) would, so keep it absent on sweep hosts. How the runner uses it: - `codex_job.py` sets `CODEX_HOME` to that home, drops `--ignore-user-config` and adds `-p stack-worker`: `codex exec -p stack-worker --skip-git-repo-check -s read-only -m cx/gpt-6-astra -c model_reasoning_effort="max" -c web_search="live" ...`. diff --git a/tools/sota-convergence/landscape-sweep/build_args.py b/tools/sota-convergence/landscape-sweep/build_args.py index 528f2fbb4..0da827a7f 100755 --- a/tools/sota-convergence/landscape-sweep/build_args.py +++ b/tools/sota-convergence/landscape-sweep/build_args.py @@ -82,7 +82,8 @@ CODEX_TEMPLATE = Path("adoption/templates/codex.config.template.toml") STACK_WORKER_PROFILE = Path("adoption/templates/codex.stack-worker.config.toml") TABLE_HEADER = re.compile(r"^\s*\[") -MCP_TABLE_HEADER = re.compile(r"^\s*\[mcp_servers\.([A-Za-z0-9_-]+)(?:\.[^\]]+)?\]\s*(?:#.*)?$") +# [mcp_servers....] with optional spaces inside the brackets and a bare or double-quoted name. +MCP_TABLE_HEADER = re.compile(r'^\s*\[\s*mcp_servers\.(?:"([^"]+)"|([A-Za-z0-9_-]+))(?:\.[^\]]+)?\s*\]\s*(?:#.*)?$') def load_templates() -> dict: @@ -99,11 +100,27 @@ def mcp_sections(rendered: str) -> tuple[str, list[str]]: if TABLE_HEADER.match(line): match = MCP_TABLE_HEADER.match(line) keep = match is not None - if keep and match.group(1) not in names: - names.append(match.group(1)) + name = (match.group(1) or match.group(2)) if match else None + if keep and name not in names: + names.append(name) if keep: kept.append(line) - return "\n".join(kept).strip() + "\n", names + text = "\n".join(kept).strip() + "\n" + # Line extraction is checked against a real TOML parse (Python 3.11+): the extracted text must parse and name + # exactly the servers the whole rendered config names, or staging fails closed rather than dropping a server. + try: + import tomllib + except ImportError: + tomllib = None + if tomllib is not None: + try: + full = set((tomllib.loads(rendered).get("mcp_servers") or {}).keys()) + extracted = set((tomllib.loads(text).get("mcp_servers") or {}).keys()) + except tomllib.TOMLDecodeError as error: + raise ValueError(f"extracted [mcp_servers.*] tables are not valid TOML: {error}") from None + if extracted != full or set(names) != full: + raise ValueError(f"MCP extraction mismatch: template names {sorted(full)}, extracted {sorted(extracted)}") + return text, names def host_values_from_file(path: Path) -> dict: @@ -151,16 +168,27 @@ def stage_lane_home(work: Path, *, model: str, base_url: str, host: str, profile f"# {host_label!r} by tools/adoption/render_config.py. The provider key comes from ${OMNIROUTE_KEY_ENV}; it is never", "# stored here. supports_websockets stays unset (false): OmniRoute's WebSocket bridge drops the client headers", "# that carry the Codex version, which only its HTTP /v1/responses path forwards.", + "# Web search: GPT-6 Astra runs Responses Lite, which carries no hosted tools, so search reaches it only as", + "# Codex's standalone web search (web.run). A custom provider advertises that with", + "# supports_standalone_web_search (Codex config docs, config-advanced) and the standalone_web_search feature", + "# turns it on (under development in 0.157.1); the lane's parity check must show it works through OmniRoute.", + "# shell_snapshot is off: Codex's shell snapshot writes the exported environment, the provider key included,", + "# to /shell_snapshots (codex-rs/shell-command/src/shell_snapshot_exports.rs at rust-v0.157.1).", f'model = "{model}"', 'model_provider = "omniroute"', 'model_reasoning_effort = "max"', "", + "[features]", + "shell_snapshot = false", + "standalone_web_search = true", + "", "[model_providers.omniroute]", 'name = "OmniRoute"', f'base_url = "{base_url}"', f'env_key = "{OMNIROUTE_KEY_ENV}"', "requires_openai_auth = false", 'wire_api = "responses"', + "supports_standalone_web_search = true", "", servers_text, ]) @@ -406,7 +434,9 @@ def main(argv=None) -> int: raise ValueError("--omniroute-base-url must be a loopback http URL ending in /v1") profile = args.stack_worker_profile or (args.repo_root.resolve() / STACK_WORKER_PROFILE) if not Path(profile).is_file(): - raise ValueError(f"stack-worker profile {profile} does not exist; pass --stack-worker-profile") + raise ValueError(f"stack-worker profile {profile} does not exist: the default " + f"{STACK_WORKER_PROFILE} arrives with the Codex worker-lane change; until the " + "checkout has it, pass --stack-worker-profile PATH") lane = {"host": args.codex_host, "base_url": args.omniroute_base_url, "profile": Path(profile).resolve(), "require_key": args.omniroute_require_key} if args.skills_checked_at: diff --git a/tools/sota-convergence/landscape-sweep/codex_job.py b/tools/sota-convergence/landscape-sweep/codex_job.py index c0cf1e2e2..6da21a59a 100755 --- a/tools/sota-convergence/landscape-sweep/codex_job.py +++ b/tools/sota-convergence/landscape-sweep/codex_job.py @@ -308,7 +308,7 @@ def codex_env(lane: dict | None = None) -> dict: env = {key: value for key, value in os.environ.items() if not key.startswith("RUST_LOG")} if lane is not None and lane.get("codex_home") is not None: env["CODEX_HOME"] = str(lane["codex_home"]) - if not env.get(lane["api_key_env"]) and lane.get("api_key_placeholder"): + if not (env.get(lane["api_key_env"]) or "").strip() and lane.get("api_key_placeholder"): env[lane["api_key_env"]] = lane["api_key_placeholder"] # keyless loopback gateway; a real key wins return env @@ -598,7 +598,7 @@ def run(base: Path, job: str) -> int: finish(directory, EXIT_NO_CODEX) return EXIT_NO_CODEX lane = config if config["codex_home"] is not None else None - if lane is not None and not os.environ.get(lane["api_key_env"]) and not lane["api_key_placeholder"]: + if lane is not None and not (os.environ.get(lane["api_key_env"]) or "").strip() and not lane["api_key_placeholder"]: write_atomic(directory / "stderr.txt", f"{lane['api_key_env']} is not set in the runner's environment; start " "the harness with the gateway key loaded from its store by pointer (docs/secret-storage.md)\n") finish(directory, EXIT_NO_KEY) From 1e81cce0a36ce5053598e81bedd398dddeaa0a7e Mon Sep 17 00:00:00 2001 From: Scout Date: Sun, 27 Sep 2026 02:49:03 -0400 Subject: [PATCH 8/8] #387 GPT-6 re-check: the MCP extraction check compares whole tables The GPT-6 re-check of 8780d93e resolved the snapshot and search findings. It left one low finding open: after [mcp_servers.demo], an env table written as [mcp_servers.'demo'.env] or [ mcp_servers . demo . env ] was dropped silently, because the check compared server names only. - The check now compares the parsed mcp_servers table of the extraction with the whole rendered config's, nested settings included. Any difference names the servers and fails staging closed. - The header pattern accepts TOML's literal-quoted names and spaces around the dots, so those spellings are kept rather than refused. - Without tomllib (Python 3.11+), staging refuses instead of skipping the check. The failing control: both new tests fail against 8780d93e's build_args.py. The real template rendered for this host still extracts all 7 servers, env tables included, equal to the full parse. The other open medium (default worker profile absent) resolves when #389 lands the profile, and #389 merges first. Tests: 166 run, OK (3 skipped); validate.py passed. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012qwXNtFyUhY5x2dtrm7tG5 --- manifests/evidence.json | 8 ++-- tests/test_landscape_sweep_harness.py | 27 +++++++++++--- .../landscape-sweep/build_args.py | 37 ++++++++++++------- 3 files changed, 49 insertions(+), 23 deletions(-) diff --git a/manifests/evidence.json b/manifests/evidence.json index f585f906a..1061b82ae 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -38798,8 +38798,8 @@ }, { "path": "tests/test_landscape_sweep_harness.py", - "sha256": "d59fc347a50a17b68f90c786dd51d4843d78867fa55e4c60db35e7eed5a61d61", - "bytes": 160583 + "sha256": "a61c1be98cbb113e7ab574936f809dd23be2bc959c576a361153267a5c6371db", + "bytes": 162061 }, { "path": "tests/test_lane_packets.py", @@ -39283,8 +39283,8 @@ }, { "path": "tools/sota-convergence/landscape-sweep/build_args.py", - "sha256": "10dc350aaf3dc29ce727758cc8bc421480678aab8540884c32afc2e938a8339a", - "bytes": 29780 + "sha256": "5643aa8aa5fe4174457ca0c2b694a49c8c4a020a4f279420e7034f44061a6eb5", + "bytes": 30467 }, { "path": "tools/sota-convergence/landscape-sweep/build_inputs.py", diff --git a/tests/test_landscape_sweep_harness.py b/tests/test_landscape_sweep_harness.py index c2f4be3c4..29c8984ad 100644 --- a/tests/test_landscape_sweep_harness.py +++ b/tests/test_landscape_sweep_harness.py @@ -746,21 +746,38 @@ def test_codex_host_accepts_a_private_host_value_file_by_path(self): self.assertEqual(missing.returncode, 2) self.assertIn("does not exist", missing.stderr) + @unittest.skipIf(sys.version_info < (3, 11), "tomllib is Python 3.11+") def test_mcp_extraction_accepts_quoted_and_spaced_headers_and_fails_closed(self): + import tomllib text = ('model = "x"\n[mcp_servers."demo"]\ncommand = "a"\n[ mcp_servers.plain ]\ncommand = "b"\n' '[mcp_servers.plain.env]\nK = "v"\n[projects."/p"]\ntrust_level = "trusted"\n') sections, names = build_args.mcp_sections(text) self.assertEqual(names, ["demo", "plain"]) self.assertNotIn("[projects.", sections) - try: - import tomllib - except ImportError: - return - self.assertEqual(sorted(tomllib.loads(sections)["mcp_servers"]), ["demo", "plain"]) + self.assertEqual(tomllib.loads(sections)["mcp_servers"], tomllib.loads(text)["mcp_servers"]) # A server the line extraction cannot see (declared with inline-table syntax under [mcp_servers]) fails closed. with self.assertRaises(ValueError): build_args.mcp_sections('[mcp_servers]\nhidden = { command = "c" }\n[mcp_servers.plain]\ncommand = "b"\n') + @unittest.skipIf(sys.version_info < (3, 11), "tomllib is Python 3.11+") + def test_mcp_extraction_keeps_nested_tables_in_every_header_spelling(self): + import tomllib + # TOML allows a literal-quoted name and spaces around the dots; a server's env table in either spelling is kept. + for env_header in ("[mcp_servers.'demo'.env]", "[ mcp_servers . demo . env ]", "[mcp_servers . 'demo' . env]"): + text = f'[mcp_servers.demo]\ncommand = "a"\n{env_header}\nK = "v"\n[projects."/p"]\ntrust_level = "t"\n' + sections, names = build_args.mcp_sections(text) + self.assertEqual(names, ["demo"], env_header) + self.assertEqual(tomllib.loads(sections)["mcp_servers"], {"demo": {"command": "a", "env": {"K": "v"}}}, + env_header) + # A nested table the line extraction cannot see (a quoted top-level key) would lose the env: fail closed. + with self.assertRaisesRegex(ValueError, r"differ from the rendered config for \['demo'\]"): + build_args.mcp_sections('[mcp_servers.demo]\ncommand = "a"\n["mcp_servers".demo.env]\nK = "v"\n') + + def test_mcp_extraction_refuses_without_tomllib(self): + with mock.patch.dict(sys.modules, {"tomllib": None}): # a None entry makes the import raise ImportError + with self.assertRaisesRegex(ValueError, "Python 3.11"): + build_args.mcp_sections('[mcp_servers.demo]\ncommand = "a"\n') + def test_missing_profile_names_the_flag(self): work = stage_work(self) done = build(work, "--gpt6-provider", "omniroute", "--codex-host", "example", diff --git a/tools/sota-convergence/landscape-sweep/build_args.py b/tools/sota-convergence/landscape-sweep/build_args.py index 0da827a7f..7f7bae059 100755 --- a/tools/sota-convergence/landscape-sweep/build_args.py +++ b/tools/sota-convergence/landscape-sweep/build_args.py @@ -82,8 +82,10 @@ CODEX_TEMPLATE = Path("adoption/templates/codex.config.template.toml") STACK_WORKER_PROFILE = Path("adoption/templates/codex.stack-worker.config.toml") TABLE_HEADER = re.compile(r"^\s*\[") -# [mcp_servers....] with optional spaces inside the brackets and a bare or double-quoted name. -MCP_TABLE_HEADER = re.compile(r'^\s*\[\s*mcp_servers\.(?:"([^"]+)"|([A-Za-z0-9_-]+))(?:\.[^\]]+)?\s*\]\s*(?:#.*)?$') +# [mcp_servers....] as TOML writes a table header: optional spaces inside the brackets and around the dots, and +# a bare, double-quoted or single-quoted (literal) name. +MCP_TABLE_HEADER = re.compile( + r"""^\s*\[\s*mcp_servers\s*\.\s*(?:"([^"]+)"|'([^']+)'|([A-Za-z0-9_-]+))\s*(?:\.[^\]]*)?\]\s*(?:#.*)?$""") def load_templates() -> dict: @@ -100,26 +102,33 @@ def mcp_sections(rendered: str) -> tuple[str, list[str]]: if TABLE_HEADER.match(line): match = MCP_TABLE_HEADER.match(line) keep = match is not None - name = (match.group(1) or match.group(2)) if match else None + name = (match.group(1) or match.group(2) or match.group(3)) if match else None if keep and name not in names: names.append(name) if keep: kept.append(line) text = "\n".join(kept).strip() + "\n" - # Line extraction is checked against a real TOML parse (Python 3.11+): the extracted text must parse and name - # exactly the servers the whole rendered config names, or staging fails closed rather than dropping a server. + # Line extraction is checked against a real TOML parse: the extracted text must parse to exactly the + # mcp_servers table of the whole rendered config, every server and every nested setting, or staging fails closed + # rather than dropping a server or a server's env. Without tomllib (Python 3.11+) nothing can check it. try: import tomllib except ImportError: - tomllib = None - if tomllib is not None: - try: - full = set((tomllib.loads(rendered).get("mcp_servers") or {}).keys()) - extracted = set((tomllib.loads(text).get("mcp_servers") or {}).keys()) - except tomllib.TOMLDecodeError as error: - raise ValueError(f"extracted [mcp_servers.*] tables are not valid TOML: {error}") from None - if extracted != full or set(names) != full: - raise ValueError(f"MCP extraction mismatch: template names {sorted(full)}, extracted {sorted(extracted)}") + raise ValueError("checking the extracted [mcp_servers.*] tables needs Python 3.11+ (tomllib)") from None + try: + full = tomllib.loads(rendered).get("mcp_servers") or {} + except tomllib.TOMLDecodeError as error: + raise ValueError(f"the rendered Codex config is not valid TOML: {error}") from None + if not isinstance(full, dict): + raise ValueError("the rendered Codex config's mcp_servers is not a table") + try: + extracted = tomllib.loads(text).get("mcp_servers") or {} + except tomllib.TOMLDecodeError as error: + raise ValueError(f"extracted [mcp_servers.*] tables are not valid TOML: {error}") from None + if extracted != full or set(names) != set(full): + differ = sorted(name for name in set(full) | set(extracted) | set(names) + if full.get(name) != extracted.get(name) or name not in full or name not in names) + raise ValueError(f"MCP extraction mismatch: the extracted tables differ from the rendered config for {differ}") return text, names