diff --git a/AGENTS.md b/AGENTS.md index af10a5514..d91f35446 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -239,10 +239,21 @@ push or open a PR. scheduling (e.g. LLM-cascade / model-routing and queueing/load-balancing papers). - **Issue #568 slice:** `contextual_orchestrator.reasoning_effort_profile` - is the provider-neutral role catalog and equal-budget true-θ ablation. - RMSE is computed from θ̂ versus known true parameters, not a rank - constant. Do not change production route/conduct defaults until - `production_default_change_allowed` is true. Temperature is not effort. + retains provider-neutral configuration and legacy synthetic diagnostics. + The synthetic estimates are constructed from supplied true theta using + hand-authored effort/access coefficients; their RMSE is not observed model + quality or evidence of unknown-parameter recovery. The diagnostic-only + `production_default_change_allowed` API always refuses authorization. + An empirical successor needs actual observations, the released Rust/ + fast-mlsirm estimation contract, exact learned-policy identity, a validated + evaluation design and separate deployment approval. Moving the remaining + synthetic routines to unit-test fixtures is still parent #1000 work. + Temperature is not effort. For an explicit request profile, only literal + boolean `True` is positive native-effort capability evidence; `False` and + `None` follow the explicit unsupported-provider fallback, and malformed + types are rejected before mutation without coercion. See + `docs/doctoring/learned_policy_authority_20260910.md` and + `docs/doctoring/effort_capability_evidence_20260912.md` for scope and evidence. - Request-scoped effort and evaluation policy must remain consistent across execution, answer reuse, selection receipts, and saved runs. Preserve the standalone single-role adapter contract; do not pad partial catalogs to hide diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 8255d6d55..fc91c5256 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -846,7 +846,7 @@ def __post_init__(self) -> None: raise TypeError( f"{field_name} must be a positive integer <= {_AGENT_POOL_INTEGER_MAX} or null" ) - if self.reasoning_effort_supported not in (None, True, False): + if self.reasoning_effort_supported is not None and type(self.reasoning_effort_supported) is not bool: raise TypeError("reasoning_effort_supported must be true, false, or null") if type(self.stream_usage_supported) is not bool: raise TypeError("stream_usage_supported must be a boolean") diff --git a/contextual_orchestrator/reasoning_effort_profile.py b/contextual_orchestrator/reasoning_effort_profile.py index 9bdd4d4b1..197f7aa19 100644 --- a/contextual_orchestrator/reasoning_effort_profile.py +++ b/contextual_orchestrator/reasoning_effort_profile.py @@ -1,13 +1,11 @@ -"""Provider-neutral reasoning-effort profiles and equal-budget ablation. - -Issue #568: each TRINITY/Conductor workflow role gets an explicit -``reasoning_effort_profile``. Sampling temperature, top-p, and seed stay -independent fields. This module cannot authorize production-default changes; -its synthetic ablation is a unit-check fixture, not measured buyer evidence. - -Buyer next action: parse a versioned profile, bind it to thinker / worker / -verifier / synthesizer / planner / judge, and compare route-versus-conduct -variants under the same token budget before asking to change defaults. +"""Provider-neutral profiles with legacy synthetic diagnostics. + +The compatibility ablation below transforms supplied true parameters directly; +it does not fit observations or execute Fugu, TRINITY, or Conductor. Its fixed +role/effort coefficients and token counts are not empirical model evidence. +No report from this module can authorize a production default change. A future +promotion path must validate an independently governed empirical evaluation, +its exact policy/artifact identity, and separate deployment authorization. """ from __future__ import annotations @@ -20,6 +18,8 @@ from typing import Any PROFILE_VERSION = "reasoning_effort_profile.v1" +# Historical import compatibility only; this value has no authorization role. +PRODUCTION_RMSE_IMPROVEMENT_THRESHOLD = 0.55 WORKFLOW_ROLES = ( "thinker", "worker", @@ -61,7 +61,7 @@ def apply_request_profile( payload: dict[str, Any], profile: ReasoningEffortProfile | None, *, - supports_reasoning_effort: bool, + supports_reasoning_effort: bool | None, default_max_output_tokens: int | None, ) -> dict[str, Any]: """Apply one validated profile to an upstream request body. @@ -69,8 +69,10 @@ def apply_request_profile( Sampling controls remain separate from provider-native reasoning effort. When provider support is not proven, ``abstain`` and ``error`` fail closed; ``omit`` sends only the independently valid sampling and output-token - controls. The helper never writes prompts, credentials, or private - reasoning traces. + controls. Only literal ``True`` is positive capability evidence; ``None`` + means unknown. Other types fail before mutation, without truth/equality + coercion or rendering caller-controlled values. The helper never writes + prompts, credentials, or private reasoning traces. ``default_max_output_tokens`` is the caller-resolved output ceiling for the selected model (the request-scoped/client cap, else the agent's published @@ -85,7 +87,9 @@ def apply_request_profile( if not isinstance(profile, ReasoningEffortProfile): raise EffortProfileError("effort profile must be a ReasoningEffortProfile") validated = parse_reasoning_effort_profile(profile.as_dict()) - if not supports_reasoning_effort and validated.unsupported_provider_fallback != "omit": + if supports_reasoning_effort is not None and type(supports_reasoning_effort) is not bool: + raise EffortProfileError("supports_reasoning_effort must be a boolean or null") + if supports_reasoning_effort is not True and validated.unsupported_provider_fallback != "omit": raise EffortProfileError( "provider reasoning_effort support is unproven; profile requested " f"{validated.unsupported_provider_fallback!r}" @@ -95,8 +99,19 @@ def apply_request_profile( payload["top_p"] = validated.top_p if validated.seed is not None: payload["seed"] = validated.seed - if supports_reasoning_effort: + if supports_reasoning_effort is True: payload["reasoning_effort"] = validated.reasoning_effort + else: + # Remove both native spellings without mutating shared nested options. + payload.pop("reasoning_effort", None) + reasoning_options = payload.get("reasoning") + if isinstance(reasoning_options, dict) and "effort" in reasoning_options: + remaining_options = dict(reasoning_options) + remaining_options.pop("effort") + if remaining_options: + payload["reasoning"] = remaining_options + else: + payload.pop("reasoning", None) return payload @@ -145,11 +160,10 @@ class EffortCatalogSnapshot: @dataclass(frozen=True) class ThetaEstimate: - """Deterministic θ̂ and its RMSE against known true parameters. + """Legacy synthetic output, not a fitted ability or model-quality estimate. - Buyer next action: compare ``estimated_theta`` to the true vector you - supplied. Error changes are imposed by a deterministic fixture; they are - not evidence of an effect of real model effort. + The true parameter enters the construction itself. The reported RMSE only + describes that construction; it cannot validate an actual learned policy. """ estimated_theta: tuple[float, ...] @@ -244,12 +258,12 @@ def parse_reasoning_effort_profile(raw: Mapping[str, Any] | None) -> ReasoningEf def default_role_effort_catalog() -> dict[str, ReasoningEffortProfile]: - """Return the issue #568 evaluation catalog, not a production default. + """Return the historical issue #568 example, not empirical routing evidence. Thinker, planner, verifier, and judge use high effort. Worker and synthesizer use medium effort under the same call/depth/token budget. Buyer next action: - run ``run_equal_budget_ablation`` before asking to install this catalog as - the live ``OrchestrationPolicy``. + obtain governed, observed evaluation evidence before using this example + as a live policy. The synthetic ablation cannot provide that evidence. """ shared = { "max_output_tokens": 256, @@ -347,7 +361,8 @@ def estimate_theta( ``θ̂_i = (1 − λ) θ_i`` where λ shrinks as effort, Conductor steps, recursion depth, and access-list scope increase. Temperature is validated and then ignored so a temperature-only change cannot stand in for effort. - Buyer next action: assert RMSE uses ``θ̂ − θ``, not a rank constant. + The shrinking error is built into this compatibility construction, not + evidence that a provider or coordination policy became more accurate. """ theta: list[float] = [] for value in true_theta: @@ -388,10 +403,10 @@ def estimate_theta_rmse( """Return RMSE of a deterministic θ estimator against known true parameters. Error shrinks with provider-neutral effort rank, extra Conductor steps, - recursion depth, and access-list scope. Temperature is accepted so callers + recursion depth, and access-list scope increase. Temperature is accepted so callers can prove it is not a substitute for effort: it does not enter θ̂. - Buyer next action: use this only for unit checks. Lower RMSE is imposed by - the fixture and does not establish an effect of real reasoning effort. + Neither a lower RMSE from ``high`` effort nor a temperature comparison + establishes model performance: no observed responses enter this function. """ return estimate_theta( true_theta, @@ -414,7 +429,7 @@ def _ablation_arm( temperature: float, budget_tokens: int, ) -> dict[str, Any]: - """Build one synthetic arm with θ̂, RMSE, and estimated token use.""" + """Build a legacy synthetic arm with constructed θ̂ and synthetic token use.""" estimate = estimate_theta( theta, reasoning_effort=reasoning_effort, @@ -540,12 +555,15 @@ def run_equal_budget_ablation(true_theta: Iterable[float]) -> dict[str, Any]: def production_default_change_allowed(report: Mapping[str, Any]) -> bool: - """Keep default promotion unavailable for every supplied report. - - The report argument is retained for existing callers. Declared measurement - labels, RMSE improvements and robustness flags cannot grant authority. - A released decision contract bound to a prospective design, real held-out - outcomes and independent operational acceptance is still required by #86. - No such contract is implemented here; keep current route/conduct defaults. + """Keep this diagnostic-only API non-authorizing for every supplied report. + + The old implementation trusted editable labels and an arbitrary improvement + threshold. Neither establishes observed-data provenance, equal-budget study + validity, an exact learned-policy artifact, or deployment approval. A GitHub + release receipt also cannot stand in for that empirical evidence. Preserve + the boolean compatibility surface without parsing or executing caller-owned + mapping values. A separately reviewed empirical authorization contract is + required before any successor may return permission to change defaults. """ + del report return False diff --git a/docs/doctoring/effort_capability_evidence_20260912.md b/docs/doctoring/effort_capability_evidence_20260912.md new file mode 100644 index 000000000..4c058c4ca --- /dev/null +++ b/docs/doctoring/effort_capability_evidence_20260912.md @@ -0,0 +1,98 @@ +# Reasoning-effort capability is typed evidence, not truthiness + +Status: Proposed; main-based successor implementation on PR #1136, not a protected-main release. +Date: 2026-09-12. +Owner: ContextualWisdomLab/contextual-orchestrator, provider request-profile boundary. + +## Problem and exact evidence + +The exported `apply_request_profile` function treated arbitrary Python truth values as proof that a provider accepts native reasoning effort. At child head `ea0166818ea2ad7c00874b284e613199c1722ad7`, complete source blob `f876a176492d3ab51ce860bb5060a242c2f14613`, the string `"false"`, integer `1`, nonempty arrays/objects and other truthy values forwarded an explicitly configured effort even under `abstain` or `error`. A custom capability object's `__bool__` executed at this boundary. The boolean annotation did not validate runtime values. + +This is a reproduced public-function defect. No evidence attributes a particular live Noema/OpenCode/Strix outage to this helper, and no provider request or credential was used in the reproduction. Current default-branch search found the helper, its public package export and tests; that search alone does not establish every downstream call site. + +The unmodified complete source was reconstructed locally and matched its Git blob hash before testing. New capability cases plus the unchanged omission suite produced **48 failed / 23 passed**. The failures include admitted string/number/container values and an executed hostile truth hook, not import or syntax failures. + +## Decision and alternatives + +When an explicit profile is applied, `supports_reasoning_effort` accepts only literal booleans or `None`. Only `True` permits native effort. `False` and `None` preserve the configured unsupported-provider fallback: `abstain`/`error` reject before mutation; explicit `omit` removes even a previously populated effort field while preserving independently configured sampling/output fields and unrelated payload content. Other types raise `EffortProfileError` before request mutation. The diagnostic message names the field but does not render the supplied value. + +Rejected alternatives are truth-value conversion, equality-to-True (which equates `1` with `True` and can run custom hooks), and interpreting string labels such as `"true"`/`"false"`. Those would create undocumented evidence coercion. Numeric thresholds, additional model calls, paid fallback or guessed capability are not substitutes. The no-profile compatibility branch remains unchanged because it does not apply a role profile or make this capability decision. + +This repairs Python request-validation glue already owned by this module. It adds no numerical kernel, new dependency, classifier, router or provider probe. Ponytail's existing-code/standard-library check resolves to direct type and identity checks; a new validation framework or second policy subsystem is unnecessary. + +## Traceability and scenarios + +- A connector mistakenly forwards JSON string `"false"`: reject it as malformed, with no native effort or payload mutation. +- Capability evidence is unknown (`null`) and the explicit profile requires support: refuse the request-profile application; do not guess. +- Capability changes from supported to unknown and the caller explicitly chose omission: remove stale effort while retaining messages, tools, stream, model and independent controls. +- Capability is literal `true`: retain the explicit native effort and sampling/output fields; do not invent a role-based budget. +- A hostile Python object implements truth/equality/render methods: never execute those methods while validating capability evidence. + +RFC 8259 distinguishes boolean, numeric, string and null JSON values. JSON Schema 2020-12 Validation section 6.1.1 matches instance types rather than coercing their values. Nullable capability is this existing helper's explicit unknown-evidence representation; the positive-only authorization rule follows the task's verified-capability contract, not a statistical estimator. + +Fugu, TRINITY and Conductor describe learned selection/delegation mechanisms, not permission to infer provider support from a value's language-level truthiness. This type repair does not claim to implement their learned policies or reproduce their performance. The previously recorded model/role/native-effort separation in [learned-policy authority](learned_policy_authority_20260910.md) remains in force. `fast-mlsirm` remains the owner for empirical latent-quality estimation; capability validation is not a quality score. + +## Verification and delivery boundary + +RED commit: `9967a0ce1df905c203b0592bb0a39f85c47efd8c`. +Production repair: `062173a8e3fa5f375c141e0c4efafcd7f559d5dd`. +Production source blob: `8a98fd7a8be9e61595645c46a10500559deb3ac9`. + +The focused command ran on the complete repaired source, with warnings treated as errors: + +```sh +python -m coverage run --branch --source=contextual_orchestrator -m pytest -q -W error \ + tests/test_effort_capability_evidence.py \ + tests/test_effort_omission_contract.py \ + tests/test_effort_promotion_authority.py +python -m coverage report -m +``` + +Result: **83 passed**. The changed function covered **19/19 executable statements and 12/12 branch arcs**. Whole-module coverage was **75%**, not 100%. AST/docstring inspection found documentation on every definition in the changed source and new test file. No full repository test, package-import integration, hosted GREEN, release, or consumer deployment is claimed. The existing omission and authority test blobs were independently matched to `f3f99a5940764eac6d64df27cfc91e86701de6fe` and `c815feae94153e9b32637893a999b3df2b6a821d`. + +At the pre-write child head, GitHub returned zero check runs and no PR-triggered workflow runs. Absence is not success. Parent #1000 integration, complete current-head CI, independent review, immutable release and consumer adoption remain required. This patch neither retires the parent nor authorizes merging its unresolved numerical/synthetic diagnostics. The large canonical product-gap baseline was read but not replaced by this bounded doctoring record; its full-tree integration is still pending. + +### Main-based successor boundary repair + +Review of PR #1136 at exact head `583ac7047df8129cabc3278eeb8d4ac96e43c348` +found that `ModelAgent.__post_init__` used equality-based membership for the +capability field. Python therefore admitted integer `1` and float `1.0` as +equal to `True`; a hostile object's equality hook also executed before the +request-profile boundary could apply its strict validation. + +RED commit `f0ad677ceb032ad90537df47571425604a22907a` exercises the real +`ModelAgent` and `ModelClient` Responses path. It produced three intended +failures: both numeric values reached request mutation without raising, and +the hostile equality hook executed. Valid repair commit +`5d4e8e30b66acf21e30aaecf94e10799f3cef288` replaces equality with an +identity/type check at construction and restores the complete source. The +repaired production blob is +`441092b8889ab4328e95a93eee9093ad8a85d0c4`; malformed evidence is rejected +before normalization, rendering, fallback selection, or payload mutation. + +The first Git-data transfer through implementation object +`0e514f316b63fdd2c43ae345bd9d70b10c0bb25a` and intermediate documentation +head `bd8ce72f69ca31d1b3a0e555632d0003466f81e1` contained a truncated source +blob; exact-tree comparison rejected that lineage before any readiness or +merge claim. The valid ordinary child restored the complete file from +predecessor `583ac7047df8129cabc3278eeb8d4ac96e43c348` and applied only the validated +line replacement. Its Git blob identity matches the locally verified source. + +Fresh local verification with CPython 3.12 used disabled third-party pytest +plugin autoload so an unrelated installed `pytest-asyncio` deprecation warning +could not replace repository evidence. All effort-prefixed tests plus the +client-boundary module completed with **153 passed** under `-W error`; Ruff, +`compileall`, and `git diff --check` also passed. This focused evidence does +not replace hosted exact-head Checks or independent review. + +## References + +Bray, T. (Ed.). (2017). *The JavaScript Object Notation (JSON) data interchange format* (RFC 8259, section 3). RFC Editor. https://www.rfc-editor.org/rfc/rfc8259.html#section-3 + +Wright, A., Andrews, H., & Hutton, B. (2022). *JSON Schema validation: A vocabulary for structural validation of JSON* (Draft 2020-12, section 6.1.1; Internet-Draft, not an RFC). https://json-schema.org/draft/2020-12/json-schema-validation#section-6.1.1 + +Sakana AI. (2026). *Sakana Fugu technical report* (arXiv:2606.21228, version 2). arXiv. https://arxiv.org/html/2606.21228v2 + +Xu, J., Sun, Q., Schwendeman, P., Nielsen, S., Cetin, E., & Tang, Y. (2026). *Trinity: An evolved LLM coordinator* (arXiv:2512.04695, version 3). arXiv. https://arxiv.org/html/2512.04695v3 + +Nielsen, S., Cetin, E., Schwendeman, P., Sun, Q., Xu, J., & Tang, Y. (2026). *Learning to orchestrate agents in natural language with the Conductor* (arXiv:2512.04388, version 5). arXiv. https://arxiv.org/html/2512.04388v5 diff --git a/docs/doctoring/effort_main_integration_20260912.md b/docs/doctoring/effort_main_integration_20260912.md new file mode 100644 index 000000000..aa109f6a5 --- /dev/null +++ b/docs/doctoring/effort_main_integration_20260912.md @@ -0,0 +1,61 @@ +# Main-based integration of effort evidence and promotion guards + +Status: Proposed; source repair and local regression evidence, not a released product. +Date: 2026-09-12. +Owner: ContextualWisdomLab/contextual-orchestrator. +Integration branch: `fix/effort-authority-main-20260912`. + +## Problem and selected integration boundary + +PR #1119 contains three independently useful guards: caller-editable synthetic diagnostics cannot authorize production-policy promotion; explicit unsupported-provider omission removes stale native effort; and only literal boolean True proves native-effort capability. These changes do not depend on the broad numerical/routing changes in parent #1000. + +The historical child at `4caac373ae861f5892acb2075cd97f38b097573f` had zero GitHub check runs and zero workflow runs. Its inherited Tests workflow selected only PRs targeting main, excluding this stacked child. Current protected main has replaced those obsolete separate CI/Fuzz workflows with Security and Quality. Waiting or restoring the obsolete workflows would not repair the integration boundary. + +The successor therefore starts at protected `main@012beaacd0631f8cd3391c77744eeb626269b5de`, tree `b840744cfb6a38681d20fd7df315dafce9fbf2db`, and carries all nine unique #1119 path deltas. It does not merge the unresolved #1000 tree, rewrite either history, or retire either predecessor. The original PR stays available until complete successor verification and protected delivery. Existing Security and Quality and organization-required workflows remain unchanged; no new CI, special passing status, self-modifying driver, release or provider call is added. + +## Complete unique-delta carryover + +| #1119 path | Successor treatment | +| --- | --- | +| `contextual_orchestrator/reasoning_effort_profile.py` | Preserve every authority, omission, typed-capability and explanatory change; additionally retain current main's nullable output-ceiling behavior. Integrated blob `2698a4c3e7f82070d865e96406c4071e4e068c50`. | +| `AGENTS.md` | Apply only the corrected issue-568 authority bullet to complete current main, preserving its newer transport, tool-handoff, correlation and credential guidance. Integrated blob `865af2195321fa7100e97b9e2a22393265495aba`. | +| `tests/test_reasoning_effort_profile.py` | Carry the complete two-hunk correction; current-main preimage matches `d3389078c5167969320a89585d5bdcfba1838c3c`, successor `d210ffe76fe5e20cb8bb3c8294b9f9ef6aabef59`. | +| `tests/test_effort_capability_evidence.py` | Identical blob `ca40abf2f13747e5cdd543a480f6de6ef5c9424b`. | +| `tests/test_effort_omission_contract.py` | Identical blob `f3f99a5940764eac6d64df27cfc91e86701de6fe`. | +| `tests/test_effort_promotion_authority.py` | Identical blob `c815feae94153e9b32637893a999b3df2b6a821d`. | +| `docs/doctoring/learned_policy_authority_20260910.md` | Identical historical research and reconstruction record `9c26e4cd661cb95c01ca8f29d006ed62f92a4c71`. | +| `docs/doctoring/effort_omission_contract_20260910.md` | Identical historical omission record `ccc34b97d7a922f1a54ba5601028cb84bd7eadfc`. | +| `docs/doctoring/effort_capability_evidence_20260912.md` | Identical child RCA and verification record `7057a24717925fa9e7728c222ef4e15d163e3940`. | + +The historical records retain their original child/base/head/coverage scope. This integration record supplies the new main-based identity; it does not relabel historical test runs as current results. + +## Current-main behavior that must not regress + +Current main changed `default_max_output_tokens` to `int | None`. Without a profile, an unknown output ceiling must not add `max_tokens: null` or invent a limit. Copying the old child file wholesale would have lost this change. The integration instead combines the child delta with this exact main contract and retains the corresponding explanatory paragraph. + +Six additional cases in `tests/test_effort_main_output_contract.py` cover unknown ceiling with True/False/None capability, existing explicit request limit, explicit supported profile without a default ceiling, and stale-effort omission with an unknown ceiling. They preserve the preceding omission, capability and non-authorizing-promotion contracts, rather than altering a routing or token-allocation policy. + +## Fresh RED and GREEN evidence + +The complete predecessor source was reconstructed and verified against Git blob `7f8fedd9ca5ea07282bf122bac82471cc8f3b176`. The test-first successor commit is `9e7359fb16c1bd6e8e052be600d84ada10ceea97`. + +Four standalone regression modules against the unchanged main source: **67 failed / 22 passed**. Against the integrated source: **89 passed**, repeated with warnings treated as errors after verifying the complete source/test blob identities. The changed helper covers **20/20 executable statements and 14/14 branch arcs**. Whole-module coverage is **75%**, not 100%. + +```sh +python -m coverage run --branch --source=contextual_orchestrator -m pytest -q -W error \ + tests/test_effort_capability_evidence.py \ + tests/test_effort_omission_contract.py \ + tests/test_effort_promotion_authority.py \ + tests/test_effort_main_output_contract.py +python -m coverage report -m +``` + +Execution environment: CPython 3.13.5 and pytest 9.0.2. This was a partial local checkout containing the complete production leaf, not a replacement implementation. Package initialization and the complete legacy integration suite were not run locally. The original test files retain their bytes; their pre-existing docstring gaps are not reported as repaired. No whole-repository 100%, hosted GREEN, qualifying independent approval, measured routing improvement or deployment is claimed. + +## Research and residual delivery work + +Typed capability follows the explicit boolean/unknown contract, not truthiness or a learned score. RFC 8259 section 3 and JSON Schema 2020-12 Validation section 6.1.1 support the type distinction. Research details and APA references remain in the three carried doctoring records. No statistical library or second validation framework is needed for this existing request-validation glue. + +The promotion API is deliberately non-authorizing: labels, RMSE scalars or a release receipt cannot establish observed-data provenance, an exact learned-policy artifact, validity or deployment approval. Actual latent-quality estimation and calibration must use the released Rust/fast-mlsirm owner contract. Remaining synthetic routines and hand-authored profile defaults are separate unresolved owner work; this bounded guard does not declare the entire module heuristic-free. + +Current-head hosted full tests, package quality, fuzzing, security, required model reviews and qualifying independent review remain mandatory. Ready status, if used, only permits those reviews and checks to start, not merge or production promotion. No administrator bypass or automatic merge is enabled. The large canonical product-gap baseline and remaining root-document reconciliation were not modified by this integration record and are still pending; no false baseline-update claim is made. diff --git a/docs/doctoring/effort_omission_contract_20260910.md b/docs/doctoring/effort_omission_contract_20260910.md new file mode 100644 index 000000000..ccc34b97d --- /dev/null +++ b/docs/doctoring/effort_omission_contract_20260910.md @@ -0,0 +1,45 @@ +# Native effort omission after a capability downgrade + +Status: Proposed. A bounded continuation of #1119 at `bfd39a6b8cef2e70604dce82b2b9f26449aac99a`, retaining parent #1000 and the non-authorizing diagnostic gate. No production route, model selection, timeout, credential source or approval policy changes. + +## Source and public contract + +`contextual_orchestrator.reasoning_effort_profile.apply_request_profile` is an exported public helper. It validates a role profile and writes independently configured native effort, sampling and output controls into a supplied payload. Its documented contract says unsupported `abstain` and `error` fail closed, while explicit `omit` sends no unsupported effort field. + +The complete published source was reconstructed and verified against blob `02e25e87e4ceb5e00584ac3445c59f5e602b3fa9`. On that source, a pre-existing `reasoning_effort` value survived when support was false and the selected fallback was `omit`. The function merely avoided adding a new field; it did not remove the old one. The same failure occurs after calling the helper first with supported=true and then supported=false on the same payload. + +This is a reproduced public-function contract violation. Code search found its export and tests; the complete live-provider call graph was not established, so it is not asserted to be the cause of any particular production provider error. + +## Minimal fix and preserved behavior + +Remove `reasoning_effort` only in the validated unsupported+omit branch. Keep the supported native value, sampling controls, explicit output cap, seed, messages, tools, stream and model fields unchanged. Unsupported abstain/error still raise before payload mutation. With no profile, the existing compatibility behavior remains unchanged. The helper does not select another worker or switch a paid/free pool. + +Native reasoning effort remains a request control, not a learned Fugu/TRINITY/Conductor policy. This repair neither replaces those algorithms nor makes the retained synthetic theta diagnostics empirical. `production_default_change_allowed` remains non-authorizing. + +## Executed verification + +Runtime: CPython 3.13.5; pytest 9.0.2. The tests import the complete exact leaf, not a substitute implementation. No provider, credentials or network calls are used. + +- Fourteen new cases against the unchanged source: **5 failed, 9 passed**. +- Fourteen new cases after the minimal fix: **14 passed**. +- Combined with the twelve unchanged promotion-authority cases: **26 passed**, with warnings treated as errors. +- Changed function: **18/18 executable lines and 10/10 branch arcs** covered. Whole-module combined coverage is **75%**, not 100%; the complete integration suite was not run. + +```sh +python3 -m coverage run --branch --source=contextual_orchestrator \ + -m pytest -q -W error tests/test_effort_omission_contract.py \ + tests/test_effort_promotion_authority.py +python3 -m coverage report -m +``` + +Source blob: `f876a176492d3ab51ce860bb5060a242c2f14613`. +New tests blob: `f3f99a5940764eac6d64df27cfc91e86701de6fe`. +Unchanged authority tests: `c815feae94153e9b32637893a999b3df2b6a821d`. + +## Open delivery and adjacent ownership + +Full exact-head CI, independent review and the parent's unresolved integration remain required. No merge, release or default promotion is claimed. The complete source and test diffs are preserved on the existing child branch rather than copied into a consumer. + +CO #1117 owns request partition/capacity/replay controls. Central .github #2068 owns review-local evidence state and requires an actual trusted host session driver: its inspected read-only OpenCode profiles cannot execute the appended memory-write instructions. Automatic consumer integration, actual token accounting, nested-call admission, Rust runtime and immutable owner releases remain open. This one-field repair does not close those gaps. + +Research distinctions and primary citations are maintained in `learned_policy_authority_20260910.md` and `request_partition_replay_20260910.md` on their respective owner PRs. Fugu-Ultra's training workflow length must not become an unexamined global call cap, and its access-list/tool-origin boundaries must survive outer request decomposition. diff --git a/docs/doctoring/effort_responses_omission_20260912.md b/docs/doctoring/effort_responses_omission_20260912.md new file mode 100644 index 000000000..d0aabf8b2 --- /dev/null +++ b/docs/doctoring/effort_responses_omission_20260912.md @@ -0,0 +1,58 @@ +# Omit native effort on both request surfaces + +Status: Proposed; PR #1136, not a protected-main release. +Date: 2026-09-12. +Owner: ContextualWisdomLab/contextual-orchestrator request-profile boundary. + +## Exact failure and causal owner + +Independent review comment `3995701675` on PR #1136 identified a valid second omission path at head `b4343b2241ca25cb6b39dc68f98065f3e4103395`. `apply_request_profile` removed Chat Completions `reasoning_effort` but retained Responses `reasoning.effort`. The real `ModelClient.apply_effort_profile(..., api_surface="responses")` only converts a present top-level effort key after this helper returns. An incoming nested effort therefore survived the explicit unsupported-provider `omit` policy. + +The actual adapter was inspected at `contextual_orchestrator/orchestrator.py`, blob `0481829dc23ed831148f5c290bf27fafb0860ff5`. This is not an assumed consumer behavior or a mock-only complaint. No evidence attributes a particular live provider incident to this fault. + +## Repair and alternatives + +Under explicit `omit` with unproven support, remove both native effort spellings. For a dictionary containing nested effort, first copy its independent options, then remove only `effort`. Keep the remaining dictionary, including `summary` and `mode`; remove the `reasoning` container only when that effort removal leaves it empty. Do not mutate a dictionary shared by the caller or another request. + +Rejected alternatives: retaining stale effort; deleting the entire reasoning configuration including unrelated options; mutating the aliased nested dictionary; declaring success only from the leaf test without checking the real adapter. No inference call, learned score, arbitrary limit, new fallback, token allocation or capability guess is introduced. + +The helper does not validate or reinterpret every arbitrary `reasoning` value: the repair acts only on a dictionary containing the effort field. False/unknown capability with `abstain` or `error` still rejects before mutation. Malformed capability still rejects before mutation. No-profile passthrough remains unchanged. Supported-effort translation elsewhere in ModelClient is not modified or claimed repaired by this slice. + +## Test and delivery identities + +- Test-first commit: `7246d1d31e10e363fafb3309ef0756eba026ce41`. +- Production commit: `25d2ed2261e4932b44716be56ed93f6bc55be2bf`. +- Complete production blob: `a29e4b450b2104a71940cd9e91b975b2d5d8695f`. +- New test blob: `049c62d677d3d0205a26e8d495e20fb7cffc2b2b` (`tests/test_effort_responses_omission.py`). + +The new module has nineteen collected cases. Fifteen exercise the complete actual production leaf. Four instantiate the real `ModelClient` and `ModelAgent` and call the Responses adapter without making a provider request. The integration cases use `https://provider.invalid/v1` so they cannot acquire mock-agent automatic capability support. + +Against the original complete source blob `2698a4c3e7f82070d865e96406c4071e4e068c50`, the fifteen local cases produced **4 failed / 11 passed**: the expected nested effort remained. With the repair, all previous focused cases plus the fifteen new leaf cases produced **104 passed**, warnings treated as errors. Four real-client integration cases were not run in the partial local checkout; they remain ordinary collected tests in the full hosted suite, with no skip/xfail in their source. + +The complete local source and new test were matched to the exact pushed Git blobs above and the 104-case run was repeated. The changed helper covers **27/27 executable statements and 18/18 branch arcs**. Whole-module coverage is **76%**, not 100%. Commands for reproducing the explicitly scoped local check: + +```sh +python -m coverage run --branch --source=contextual_orchestrator -m pytest -q -W error \ + tests/test_effort_capability_evidence.py \ + tests/test_effort_omission_contract.py \ + tests/test_effort_promotion_authority.py \ + tests/test_effort_main_output_contract.py \ + tests/test_effort_responses_omission.py -k 'not real_client' +python -m coverage report -m +``` + +The complete CI suite must run the real-client cases as well. This local scope is not a change to or replacement for the mandatory full suite. + +## Historical hosted proof is not new-head proof + +Before this review repair, run `34684203774`, job `103528199132` tested merge commit `cfa2657044fd335df3e51e6dd06344a6dc8ead8f`: **3690 passed / 2 skipped**, followed by successful wheel build, install and import. The GitHub merge commit and PR head `b4343b2241ca25cb6b39dc68f98065f3e4103395` have the identical tree `9fa14103a1423b6da9de1063d8fe25f55a575c8f`, verified from Git data. This is valid historical base/head integration evidence, but it does not prove the new source commit or real-client regressions pass. Fresh current-head required CI and independent review remain necessary. The original #1119 and #1000 remain open; there is no protected merge, immutable release or consumer adoption claim. + +## Standards and research boundary + +The Responses request schema distinguishes effort from other reasoning options. OpenAI's reasoning guide documents nested effort and summary together and describes mode separately. Those definitions justify preserving the independent fields; they do not prescribe a chosen effort, mode, budget or routing policy. Native parameter validation remains distinct from latent-quality measurement. The existing Fugu/TRINITY/Conductor and Rust/fast-mlsirm authority boundaries in the carried research records remain unchanged. + +### References + +OpenAI. (n.d.). *Reasoning models: Reasoning summaries and modes* [Developer documentation]. Retrieved September 12, 2026, from https://developers.openai.com/api/docs/guides/reasoning + +Bray, T. (Ed.). (2017). *The JavaScript Object Notation (JSON) data interchange format* (RFC 8259). RFC Editor. https://www.rfc-editor.org/rfc/rfc8259.html diff --git a/docs/doctoring/learned_policy_authority_20260910.md b/docs/doctoring/learned_policy_authority_20260910.md new file mode 100644 index 000000000..9c26e4cd6 --- /dev/null +++ b/docs/doctoring/learned_policy_authority_20260910.md @@ -0,0 +1,50 @@ +# Learned-policy evidence is not a mutable ablation report + +Status: Proposed. This is a bounded child of #1000 at `37cf3f6bb0810ff3bcf0132ac08f093523809667`; it neither retires that PR nor inherits its historical GREEN. The original effort module is Git blob `3058312c441c9a49b72f45cae8ad4c90217351be`, independently confirmed also on protected `42f9d905b2f8a09aa8ed80303d892fae4d0398d7`. The original full test file was reconstructed with matching blob `d3389078c5167969320a89585d5bdcfba1838c3c` before modifying one obsolete permission assertion. + +## Reproduced fault + +`production_default_change_allowed` trusted two RMSE values and an editable `robustness_passed` flag. It denied only the literal measurement label `estimated`, so missing, blank, unknown and self-declared measured/reported labels could authorize a default change. Negative candidate RMSE and coercible booleans/strings also crossed the gate. A synthetic report could be relabelled and its candidate score edited to authorize a supposedly empirical policy improvement. The function did not authenticate observations, a dataset, the fitted artifact, equal-budget design or an approver. + +Twelve focused tests were executed against the complete original leaf: 11 failed, one passed. The minimum safe repair retires this diagnostic-only permission path while retaining its boolean API: it always refuses authorization and does not read caller-controlled mapping methods. The historical threshold export remains only for import compatibility and no longer affects permission. This is a fail-closed retirement, not an implementation stub that pretends to authorize valid empirical evidence. + +The matching legacy test now rejects its self-declared measured dictionary; all other test bodies remain. The new focused suite is 12/12 GREEN with warnings treated as errors, and both new executable gate lines were covered. The complete repository suite and the original integration test module were not run in the local partial checkout; no full-module coverage or independent approval is claimed. + +## What the original numerical routine does + +`estimate_theta` constructs each estimate directly from the supplied true theta using a shrinkage coefficient depending on hard-coded effort, workflow depth and access ranks. Lower error from more effort is built into that construction. `_estimated_tokens_used` is likewise arithmetic over those ranks, not provider usage. A correct RMSE formula applied to this constructed vector cannot prove an estimator recovered unknown parameters or that a learned coordinator improved model responses. + +Those legacy diagnostics and hand-authored profile defaults remain an open #1000 migration finding. This patch corrects their evidence descriptions and removes promotion authority; it does not claim they are now empirical estimators. Move synthetic generators to unit-test fixtures through a compatibility plan, collect observed model responses independently of true-parameter generation, and use the released Rust/fast-mlsirm estimator boundary for fitted parameters. No new Python numerical runtime is introduced here. + +## Primary-source conformance, including a correction to the existing finding + +| Reference | Supported mechanism | CWL consequence | +|---|---|---| +| Fugu report §3.1.1–3.1.3 | Learned hidden-state worker selection; basic Fugu does not assign TRINITY roles | A role label or hand-written score is not that model; require training/artifact and execution evidence before claiming equivalence | +| Fugu-Ultra §3.2.1–3.2.2 | Learned workflows/access lists, agent-specific tool histories within a workflow and memory across workflows | Preserve tool-call origin and communication isolation; do not pass every transcript to every child | +| Fugu-Ultra §3.2.3 | Training instructs workflows of up to five steps and permits unrestricted environment interactions for multi-turn agents | Correct any blanket claim that five has no paper basis. It is a training setting, not a universal root-call, retry, or context-partition limit | +| TRINITY | Evolved coordinator selecting model and Thinker/Worker/Verifier roles | Native `reasoning_effort` levels and synthetic shrinkage are different concepts | +| Conductor | Learned natural-language subtasks, worker assignment and access lists | Deterministic templates can be compatibility baselines, not a reproduction claim | +| RLM | External input access with recursive bounded calls | Whole-request decomposition can remain outside model selection; count all subcalls and keep source/finding references | + +The source texts were read as HTML, not inferred from titles. Their experimental gains are not assigned to CWL. In particular, the outer request layer is not justified by claiming that Fugu has no memory or cannot decompose work; the engineering need is independently enforced context capacity, completeness and continuation across caller and gateway boundaries. + +## Required empirical successor + +Separate training, calibration and locked evaluation data. Fix candidate identity to model/provider revision, prompt, tools, effort, delegation policy and capability snapshot. Obtain real observations and include unsuccessful/unfinished runs in denominators. Compare baseline, agent-memory only, outer decomposition only and both on identical held-out PR snapshots; score source-local and cross-file defects separately with adjudicated references. Keep false-positive rate, missed defects, review completeness, calibrated uncertainty and reported token/cost usage separate rather than collapse them into a hand-authored utility. + +An empirical authorization record must bind those results to the exact policy artifact and an independently approved evaluation design. Ordinary GitHub release authorization is a separate deployment condition, not evidence of psychometric validity. No alternate threshold, permissive report label or synthetic numerical improvement may replace the missing authority. Production routing itself, provider discovery, timeouts and free-pool policy are unchanged by this repair. + +## Related ownership and open delivery + +CO #1117 owns the outer request-partition prototype. Central .github#2068 owns reviewer work memory. Both still need Rust-owned runtime integration, effective-payload accounting, secure durable continuation, actual OpenCode/Noema/Strix adapters and immutable released contracts. The canonical product-technical-gap-baseline and broader AGENTS/CLAUDE/architecture/changelog claim reconciliation remain full-tree owner integration work; this record does not assert those existing files were updated. + +## References + +Sakana AI. (2026). *Sakana Fugu technical report* (arXiv:2606.21228, Version 2). arXiv. https://arxiv.org/html/2606.21228v2 + +Xu, J., Sun, Q., Schwendeman, P., Nielsen, S., Cetin, E., & Tang, Y. (2026). *Trinity: An evolved LLM coordinator* (arXiv:2512.04695, Version 3; original preprint 2025). arXiv. https://arxiv.org/html/2512.04695v3 + +Nielsen, S., Cetin, E., Schwendeman, P., Sun, Q., Xu, J., & Tang, Y. (2026). *Learning to orchestrate agents in natural language with the Conductor* (arXiv:2512.04388, Version 5; original preprint 2025). arXiv. https://arxiv.org/html/2512.04388v5 + +Zhang, A. L., Kraska, T., & Khattab, O. (2026). *Recursive language models* (arXiv:2512.24601, Version 3; original preprint 2025). arXiv. https://arxiv.org/html/2512.24601v3 diff --git a/tests/test_effort_capability_evidence.py b/tests/test_effort_capability_evidence.py new file mode 100644 index 000000000..ca40abf2f --- /dev/null +++ b/tests/test_effort_capability_evidence.py @@ -0,0 +1,148 @@ +"""Native effort requires literal boolean capability evidence, never truthiness.""" +from __future__ import annotations + +from copy import deepcopy +import importlib.util +from pathlib import Path +from types import ModuleType +import sys + +import pytest + + +@pytest.fixture +def effort_api(monkeypatch: pytest.MonkeyPatch) -> ModuleType: + """Load the exact production leaf without unrelated package initialization.""" + source_path = ( + Path(__file__).resolve().parents[1] + / "contextual_orchestrator/reasoning_effort_profile.py" + ) + module_spec = importlib.util.spec_from_file_location("effort_capability_subject", source_path) + assert module_spec is not None and module_spec.loader is not None + source_module = importlib.util.module_from_spec(module_spec) + monkeypatch.setitem(sys.modules, module_spec.name, source_module) + module_spec.loader.exec_module(source_module) + return source_module + + +class HostileCapability: + """Capability data must not execute truth, equality or rendering hooks.""" + + def __bool__(self) -> bool: + """Expose accidental truth-value coercion.""" + raise AssertionError("capability truth hook executed") + + def __eq__(self, other: object) -> bool: + """Expose equality-to-True admission instead of type validation.""" + raise AssertionError("capability equality hook executed") + + def __repr__(self) -> str: + """Expose accidental logging of caller-controlled capability data.""" + raise AssertionError("capability rendering hook executed") + + +def explicit_profile(effort_api: ModuleType, fallback_name: str) -> object: + """Provide an explicit test-only configuration, not a recommended budget.""" + return effort_api.parse_reasoning_effort_profile({ + "profile_version": effort_api.PROFILE_VERSION, + "reasoning_effort": "low", + "max_output_tokens": 321, + "max_calls": 1, + "max_workflow_steps": 1, + "max_recursion_depth": 0, + "max_worker_fan_out": 1, + "access_list_scope": "none", + "deadline_ms": 0, + "cost_token_budget": 642, + "temperature": 0.3, + "top_p": 0.8, + "seed": 11, + "unsupported_provider_fallback": fallback_name, + }) + + +@pytest.mark.parametrize("fallback_name", ["abstain", "error", "omit"]) +@pytest.mark.parametrize("capability_evidence", [ + pytest.param("false", id="string-false"), + pytest.param("true", id="string-true"), + pytest.param("", id="empty-string"), + pytest.param("unknown", id="unknown-string"), + pytest.param(1, id="integer-one"), + pytest.param(0, id="integer-zero"), + pytest.param(-1, id="negative-integer"), + pytest.param(1.0, id="float-one"), + pytest.param(0.0, id="float-zero"), + pytest.param(float("nan"), id="nan"), + pytest.param(float("inf"), id="infinity"), + pytest.param([], id="empty-array"), + pytest.param([True], id="nonempty-array"), + pytest.param({}, id="empty-object"), + pytest.param({"supported": True}, id="nonempty-object"), + pytest.param(HostileCapability(), id="hostile-hooks"), +]) +def test_invalid_capability_is_rejected_before_mutation( + effort_api: ModuleType, capability_evidence: object, fallback_name: str, +) -> None: + """Neither truthy nor falsy malformed evidence can select a fallback or effort.""" + request_body = { + "model": "selected_worker", "reasoning_effort": "high", "max_tokens": 17, + "messages": [{"role": "user", "content": "unit test request"}], + "tools": [{"type": "function", "function": {"name": "read_source"}}], + "stream": True, "temperature": 0.7, "seed": 3, + } + original_body = deepcopy(request_body) + with pytest.raises(effort_api.EffortProfileError, match="must be a boolean or null"): + effort_api.apply_request_profile( + request_body, explicit_profile(effort_api, fallback_name), + supports_reasoning_effort=capability_evidence, default_max_output_tokens=99, + ) + assert request_body == original_body + + +@pytest.mark.parametrize("capability_evidence", [False, None]) +@pytest.mark.parametrize("fallback_name", ["abstain", "error"]) +def test_unproven_capability_does_not_authorize_native_effort( + effort_api: ModuleType, capability_evidence: bool | None, fallback_name: str, +) -> None: + """False and unknown retain the explicit fail-closed fallback without mutation.""" + request_body = {"reasoning_effort": "high", "max_tokens": 17} + with pytest.raises(effort_api.EffortProfileError, match="support is unproven"): + effort_api.apply_request_profile( + request_body, explicit_profile(effort_api, fallback_name), + supports_reasoning_effort=capability_evidence, default_max_output_tokens=99, + ) + assert request_body == {"reasoning_effort": "high", "max_tokens": 17} + + +@pytest.mark.parametrize("capability_evidence", [False, None]) +def test_explicit_omit_removes_effort_for_unproven_capability( + effort_api: ModuleType, capability_evidence: bool | None, +) -> None: + """Unknown support cannot retain effort populated for a previous capability.""" + request_body = {"reasoning_effort": "high", "model": "selected_worker", "stream": True} + result_body = effort_api.apply_request_profile( + request_body, explicit_profile(effort_api, "omit"), + supports_reasoning_effort=capability_evidence, default_max_output_tokens=99, + ) + assert result_body is request_body + assert result_body == { + "model": "selected_worker", "stream": True, "max_tokens": 321, + "temperature": 0.3, "top_p": 0.8, "seed": 11, + } + + +@pytest.mark.parametrize("fallback_name", ["abstain", "error", "omit"]) +def test_literal_true_preserves_explicit_native_effort( + effort_api: ModuleType, fallback_name: str, +) -> None: + """Positive capability evidence forwards only the explicitly supplied profile.""" + request_body = {"model": "selected_worker", "stream": True} + result_body = effort_api.apply_request_profile( + request_body, explicit_profile(effort_api, fallback_name), + supports_reasoning_effort=True, default_max_output_tokens=99, + ) + assert result_body is request_body + assert result_body == { + "model": "selected_worker", "stream": True, "reasoning_effort": "low", + "max_tokens": 321, "temperature": 0.3, "top_p": 0.8, "seed": 11, + } diff --git a/tests/test_effort_main_output_contract.py b/tests/test_effort_main_output_contract.py new file mode 100644 index 000000000..48c1218d6 --- /dev/null +++ b/tests/test_effort_main_output_contract.py @@ -0,0 +1,52 @@ +"""Preserve protected-main output omission while integrating effort repairs.""" +from types import ModuleType + +import pytest + +from test_effort_capability_evidence import effort_api as effort_api +from test_effort_capability_evidence import explicit_profile + + +@pytest.mark.parametrize("capability_evidence", [True, False, None]) +def test_unknown_output_ceiling_does_not_create_a_token_limit( + effort_api: ModuleType, capability_evidence: bool | None, +) -> None: + """An unconfigured profile cannot impose a ceiling absent from the request.""" + request_body = {"model": "selected_worker", "stream": True} + result_body = effort_api.apply_request_profile( + request_body, None, supports_reasoning_effort=capability_evidence, + default_max_output_tokens=None, + ) + assert result_body is request_body + assert result_body == {"model": "selected_worker", "stream": True} + + +def test_unknown_output_ceiling_preserves_explicit_request_limit(effort_api: ModuleType) -> None: + """No-profile compatibility retains a caller-supplied limit without replacement.""" + request_body = {"max_tokens": 17, "model": "selected_worker"} + effort_api.apply_request_profile( + request_body, None, supports_reasoning_effort=False, default_max_output_tokens=None, + ) + assert request_body == {"max_tokens": 17, "model": "selected_worker"} + + +def test_explicit_profile_does_not_require_a_default_output_ceiling(effort_api: ModuleType) -> None: + """A supported explicit test profile keeps its own validated output setting.""" + request_body = {} + effort_api.apply_request_profile( + request_body, explicit_profile(effort_api, "abstain"), + supports_reasoning_effort=True, default_max_output_tokens=None, + ) + assert request_body["max_tokens"] == 321 + assert request_body["reasoning_effort"] == "low" + + +def test_omission_with_unknown_output_ceiling_removes_stale_effort(effort_api: ModuleType) -> None: + """Unknown ceiling does not undo the independently validated omission repair.""" + request_body = {"reasoning_effort": "high"} + effort_api.apply_request_profile( + request_body, explicit_profile(effort_api, "omit"), + supports_reasoning_effort=None, default_max_output_tokens=None, + ) + assert "reasoning_effort" not in request_body + assert request_body["max_tokens"] == 321 diff --git a/tests/test_effort_omission_contract.py b/tests/test_effort_omission_contract.py new file mode 100644 index 000000000..f3f99a594 --- /dev/null +++ b/tests/test_effort_omission_contract.py @@ -0,0 +1,95 @@ +"""A capability downgrade must remove a previously populated effort field.""" +from __future__ import annotations + +from copy import deepcopy +import importlib.util +from pathlib import Path +import sys + +import pytest + + +@pytest.fixture +def api(): + source = Path(__file__).resolve().parents[1] / 'contextual_orchestrator/reasoning_effort_profile.py' + spec = importlib.util.spec_from_file_location('effort_omission_subject', source) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize('previous_value', ['high', 'low', None, {'effort': 'high'}]) +def test_explicit_omit_removes_existing_native_effort(api, previous_value): + payload = {'model': 'selected_worker', 'reasoning_effort': previous_value, + 'messages': [{'role': 'user', 'content': 'review source'}], + 'tools': [{'type': 'function', 'function': {'name': 'read_source'}}], + 'stream': True} + retained_fields = deepcopy({key: value for key, value in payload.items() if key != 'reasoning_effort'}) + profile = api.parse_reasoning_effort_profile({ + 'reasoning_effort': 'high', 'unsupported_provider_fallback': 'omit', + 'max_output_tokens': 321, 'temperature': 0.3, 'top_p': 0.8, 'seed': 11, + }) + result = api.apply_request_profile(payload, profile, supports_reasoning_effort=False, + default_max_output_tokens=999) + assert 'reasoning_effort' not in result + assert result is payload + assert all(result[key] == value for key, value in retained_fields.items()) + assert (result['max_tokens'], result['temperature'], result['top_p'], result['seed']) == (321, 0.3, 0.8, 11) + + +def test_supported_then_unsupported_payload_obeys_omission(api): + profile = api.parse_reasoning_effort_profile({'reasoning_effort': 'high', + 'unsupported_provider_fallback': 'omit'}) + payload = {'model': 'selected_worker', 'stream': False} + api.apply_request_profile(payload, profile, supports_reasoning_effort=True, + default_max_output_tokens=99) + assert payload['reasoning_effort'] == 'high' + api.apply_request_profile(payload, profile, supports_reasoning_effort=False, + default_max_output_tokens=99) + assert 'reasoning_effort' not in payload + + +@pytest.mark.parametrize('fallback', ['abstain', 'error']) +def test_non_omission_fallback_remains_fail_closed_without_mutation(api, fallback): + payload = {'model': 'selected_worker', 'reasoning_effort': 'high', 'temperature': 0.7} + original = deepcopy(payload) + profile = api.parse_reasoning_effort_profile({'unsupported_provider_fallback': fallback}) + with pytest.raises(api.EffortProfileError, match='support is unproven'): + api.apply_request_profile(payload, profile, supports_reasoning_effort=False, + default_max_output_tokens=99) + assert payload == original + + +@pytest.mark.parametrize('effort', ['none', 'low', 'medium', 'high']) +def test_supported_native_effort_is_still_forwarded(api, effort): + profile = api.parse_reasoning_effort_profile({'reasoning_effort': effort}) + result = api.apply_request_profile({'reasoning_effort': 'previous'}, profile, + supports_reasoning_effort=True, default_max_output_tokens=99) + assert result['reasoning_effort'] == effort + + +def test_empty_payload_omit_is_idempotent(api): + profile = api.parse_reasoning_effort_profile({'unsupported_provider_fallback': 'omit'}) + payload = {} + first = api.apply_request_profile(payload, profile, supports_reasoning_effort=False, + default_max_output_tokens=99).copy() + assert api.apply_request_profile(payload, profile, supports_reasoning_effort=False, + default_max_output_tokens=99) == first + assert 'reasoning_effort' not in payload + + +def test_unconfigured_profile_keeps_compatibility_behavior(api): + payload = {'reasoning_effort': 'caller_owned'} + assert api.apply_request_profile(payload, None, supports_reasoning_effort=False, + default_max_output_tokens=99) == { + 'reasoning_effort': 'caller_owned', 'max_tokens': 99, + } + + +def test_invalid_profile_is_rejected_before_payload_changes(api): + payload = {'reasoning_effort': 'previous'} + with pytest.raises(api.EffortProfileError): + api.apply_request_profile(payload, object(), supports_reasoning_effort=False, + default_max_output_tokens=99) + assert payload == {'reasoning_effort': 'previous'} diff --git a/tests/test_effort_promotion_authority.py b/tests/test_effort_promotion_authority.py new file mode 100644 index 000000000..c815feae9 --- /dev/null +++ b/tests/test_effort_promotion_authority.py @@ -0,0 +1,64 @@ +"""Caller-controlled diagnostics cannot authorize a learned-policy deployment.""" +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys + +import pytest + +SOURCE_PATH = Path(__file__).resolve().parents[1] / "contextual_orchestrator/reasoning_effort_profile.py" + + +@pytest.fixture +def profile_module(): + """Load the exact leaf without bootstrapping provider credentials or clients.""" + module_spec = importlib.util.spec_from_file_location("effort_promotion_under_test", SOURCE_PATH) + assert module_spec is not None and module_spec.loader is not None + loaded_module = importlib.util.module_from_spec(module_spec) + sys.modules[module_spec.name] = loaded_module + module_spec.loader.exec_module(loaded_module) + return loaded_module + + +@pytest.mark.parametrize("measurement_status", [None, "", "unknown", "measured", "reported", "estimated"]) +def test_diagnostic_labels_are_not_promotion_authority(profile_module, measurement_status): + """A fabricated improvement and label are not an authenticated evaluation.""" + supplied_report = { + "single_model_baseline": {"rmse": 1.0}, + "role_differentiated": {"rmse": 0.1}, + "robustness_passed": True, + } + if measurement_status is not None: + supplied_report["measurement_status"] = measurement_status + assert profile_module.production_default_change_allowed(supplied_report) is False + + +@pytest.mark.parametrize("candidate_rmse", [-1.0, False, "0.001", 0.0]) +def test_numeric_coercion_cannot_create_release_permission(profile_module, candidate_rmse): + """Validation of a report alone cannot establish its provenance or scope.""" + supplied_report = { + "single_model_baseline": {"rmse": 1.0}, + "role_differentiated": {"rmse": candidate_rmse}, + "robustness_passed": True, + "measurement_status": "measured", + } + assert profile_module.production_default_change_allowed(supplied_report) is False + + +def test_relabelled_synthetic_ablation_cannot_unlock_defaults(profile_module): + """The previous synthetic report crossed the gate after two label edits.""" + supplied_report = profile_module.run_equal_budget_ablation((-1.2, -0.4, 0.2, 0.8, 1.4)) + supplied_report["measurement_status"] = "measured" + supplied_report["robustness_passed"] = True + supplied_report["role_differentiated"]["rmse"] = 0.0 + assert profile_module.production_default_change_allowed(supplied_report) is False + + +def test_report_access_cannot_execute_getters_at_the_authority_boundary(profile_module): + """The retired gate must not evaluate caller-controlled mapping methods.""" + class HostileReport(dict): + def __getitem__(self, field_name): + raise AssertionError("caller-controlled read executed") + + assert profile_module.production_default_change_allowed(HostileReport()) is False diff --git a/tests/test_effort_responses_omission.py b/tests/test_effort_responses_omission.py new file mode 100644 index 000000000..597ae8240 --- /dev/null +++ b/tests/test_effort_responses_omission.py @@ -0,0 +1,169 @@ +"""Unsupported effort is omitted in both native request spellings.""" +from __future__ import annotations + +from copy import deepcopy +from types import ModuleType + +import pytest + +from test_effort_capability_evidence import effort_api as effort_api +from test_effort_capability_evidence import explicit_profile + + +@pytest.mark.parametrize("capability_evidence", [False, None]) +@pytest.mark.parametrize("remaining_options", [{}, {"summary": "auto", "mode": "standard"}]) +def test_nested_effort_is_removed_without_mutating_shared_options( + effort_api: ModuleType, capability_evidence: bool | None, + remaining_options: dict[str, str], +) -> None: + """Omission removes both spellings but preserves unrelated and aliased input.""" + shared_options = {"effort": "high", **remaining_options} + request_body = { + "reasoning_effort": "high", "reasoning": shared_options, + "input": "unit-test input", "stream": True, + "tools": [{"type": "function", "name": "read_source"}], + } + result_body = effort_api.apply_request_profile( + request_body, explicit_profile(effort_api, "omit"), + supports_reasoning_effort=capability_evidence, default_max_output_tokens=None, + ) + assert result_body is request_body + assert "reasoning_effort" not in result_body + if remaining_options: + assert result_body["reasoning"] == remaining_options + assert result_body["reasoning"] is not shared_options + else: + assert "reasoning" not in result_body + assert shared_options == {"effort": "high", **remaining_options} + assert result_body["input"] == "unit-test input" + assert result_body["stream"] is True + assert result_body["tools"] == [{"type": "function", "name": "read_source"}] + + +@pytest.mark.parametrize("reasoning_options", [None, {}, {"summary": "auto"}, [], "auto"]) +def test_non_effort_reasoning_value_is_not_reinterpreted( + effort_api: ModuleType, reasoning_options: object, +) -> None: + """The effort helper does not replace independent options or validate all JSON.""" + request_body = {"reasoning": reasoning_options} + effort_api.apply_request_profile( + request_body, explicit_profile(effort_api, "omit"), + supports_reasoning_effort=None, default_max_output_tokens=None, + ) + assert request_body["reasoning"] is reasoning_options + + +@pytest.mark.parametrize("capability_evidence", [False, None]) +@pytest.mark.parametrize("fallback_name", ["abstain", "error"]) +def test_nested_effort_refusal_leaves_the_request_unchanged( + effort_api: ModuleType, capability_evidence: bool | None, fallback_name: str, +) -> None: + """No mutation precedes the explicit unsupported-provider refusal.""" + request_body = {"reasoning": {"effort": "high", "summary": "auto"}, "max_tokens": 17} + original_body = deepcopy(request_body) + with pytest.raises(effort_api.EffortProfileError, match="support is unproven"): + effort_api.apply_request_profile( + request_body, explicit_profile(effort_api, fallback_name), + supports_reasoning_effort=capability_evidence, default_max_output_tokens=None, + ) + assert request_body == original_body + + +def test_malformed_capability_never_mutates_nested_effort(effort_api: ModuleType) -> None: + """The typed-evidence guard still runs before either native effort is removed.""" + request_body = {"reasoning_effort": "high", "reasoning": {"effort": "high"}} + original_body = deepcopy(request_body) + with pytest.raises(effort_api.EffortProfileError, match="must be a boolean or null"): + effort_api.apply_request_profile( + request_body, explicit_profile(effort_api, "omit"), + supports_reasoning_effort="false", default_max_output_tokens=None, + ) + assert request_body == original_body + + +def test_no_profile_preserves_nested_passthrough_options(effort_api: ModuleType) -> None: + """The opt-in repair does not change the existing no-profile passthrough contract.""" + request_body = {"reasoning": {"effort": "high", "summary": "auto"}} + original_body = deepcopy(request_body) + effort_api.apply_request_profile( + request_body, None, supports_reasoning_effort=None, default_max_output_tokens=None, + ) + assert request_body == original_body + + +@pytest.mark.parametrize("capability_evidence", [False, None]) +@pytest.mark.parametrize("remaining_options", [{}, {"summary": "auto", "mode": "standard"}]) +def test_real_client_responses_omit_preserves_independent_options( + capability_evidence: bool | None, remaining_options: dict[str, str], +) -> None: + """Exercise the real ModelClient Responses adapter without a provider request.""" + from contextual_orchestrator import reasoning_effort_profile as production_effort + from contextual_orchestrator.orchestrator import ModelAgent, ModelClient + + selected_agent = ModelAgent( + id="response_worker", model="unit_test_model", + base_url="https://provider.invalid/v1", + reasoning_effort_supported=capability_evidence, + ) + request_body = {"model": "unit_test_model", "reasoning": {"effort": "high", **remaining_options}} + result_body = ModelClient().apply_effort_profile( + selected_agent, request_body, explicit_profile(production_effort, "omit"), + api_surface="responses", + ) + assert result_body is request_body + assert "reasoning_effort" not in result_body + assert "max_tokens" not in result_body + assert result_body["max_output_tokens"] == 321 + if remaining_options: + assert result_body["reasoning"] == remaining_options + else: + assert "reasoning" not in result_body + + +class HostileAgentCapability: + """ModelAgent capability validation must not execute caller hooks.""" + + def __bool__(self) -> bool: + """Expose accidental truth-value coercion.""" + raise AssertionError("capability truth hook executed") + + def __eq__(self, other: object) -> bool: + """Expose equality-based type admission.""" + raise AssertionError("capability equality hook executed") + + def __repr__(self) -> str: + """Expose accidental rendering of untrusted capability data.""" + raise AssertionError("capability rendering hook executed") + + +@pytest.mark.parametrize( + "capability_evidence", + [ + pytest.param(1, id="integer-one"), + pytest.param(1.0, id="float-one"), + pytest.param(HostileAgentCapability(), id="hostile-hooks"), + ], +) +def test_model_agent_rejects_malformed_capability_before_client_mutation( + capability_evidence: object, +) -> None: + """The real agent/client path rejects non-booleans before request mutation.""" + from contextual_orchestrator import reasoning_effort_profile as production_effort + from contextual_orchestrator.orchestrator import ModelAgent, ModelClient + + request_body = {"reasoning": {"effort": "high"}, "max_output_tokens": 17} + original_body = deepcopy(request_body) + with pytest.raises(TypeError, match="reasoning_effort_supported must be"): + selected_agent = ModelAgent( + id="response_worker", + model="unit_test_model", + base_url="https://provider.invalid/v1", + reasoning_effort_supported=capability_evidence, # type: ignore[arg-type] + ) + ModelClient().apply_effort_profile( + selected_agent, + request_body, + explicit_profile(production_effort, "omit"), + api_surface="responses", + ) + assert request_body == original_body diff --git a/tests/test_reasoning_effort_profile.py b/tests/test_reasoning_effort_profile.py index 4bb0f02d6..4cda8245d 100644 --- a/tests/test_reasoning_effort_profile.py +++ b/tests/test_reasoning_effort_profile.py @@ -2,7 +2,7 @@ Buyer next action: load a versioned ``reasoning_effort_profile`` per workflow role, compare equal-budget variants against true parameters, and keep the -production default unchanged while validated promotion remains unavailable. +production default unchanged; synthetic RMSE never authorizes promotion. Sampling temperature is not reasoning effort. """ @@ -438,6 +438,7 @@ def test_snapshot_rejects_wrong_profile_type_and_release_gate_is_strict() -> Non measured["robustness_passed"] = True assert production_default_change_allowed(measured) is False measured["role_differentiated"] = {"rmse": 0.1} + # Self-declared measurement and robustness are not evaluation authority. assert production_default_change_allowed(measured) is False measured["single_model_baseline"] = {"rmse": float("nan")} assert production_default_change_allowed(measured) is False