-
Notifications
You must be signed in to change notification settings - Fork 1
test(ai): require payload literals for orchestrator mode scans #117
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| """LineageWeave delegates product-default LLM execution to auto policy.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| from pathlib import Path | ||
|
|
||
| from lineageweave import adjudication_client, post_chat, post_evaluation | ||
| from lineageweave.post_chat import ChatSourceDocument, ContextualOrchestratorPostChatClient | ||
|
|
||
|
|
||
| def test_post_evaluation_adapter_defaults_to_auto(monkeypatch) -> None: | ||
| observed: dict[str, object] = {} | ||
|
|
||
| def fake_post_json(url, payload, *, headers, timeout): | ||
| observed.update( | ||
| url=url, | ||
| payload=payload, | ||
| headers=headers, | ||
| timeout=timeout, | ||
| ) | ||
| return {"choices": [{"message": {"content": "{}"}}]} | ||
|
|
||
| monkeypatch.setattr(post_evaluation, "post_json", fake_post_json) | ||
| adapter = post_evaluation._OrchestratorCompleteAdapter( | ||
| "https://orchestrator.example.test", "inference_token" | ||
| ) | ||
| adapter.complete([{"role": "user", "content": "Evaluate this evidence."}]) | ||
|
|
||
| assert observed["payload"]["mode"] == "auto" | ||
|
|
||
|
|
||
| def test_post_evaluation_judge_uses_auto_by_default() -> None: | ||
| client = post_evaluation.ContextualOrchestratorPostEvaluationClient( | ||
| "https://orchestrator.example.test", "inference_token" | ||
| ) | ||
| assert client._judge.mode == "auto" | ||
|
|
||
|
|
||
| def test_post_chat_requests_verify_mode(monkeypatch) -> None: | ||
| """Citation chat must send verify on the wire, not a docstring mention of auto.""" | ||
|
|
||
| observed: dict[str, object] = {} | ||
|
|
||
| def fake_post_json(url, payload, *, headers, timeout): | ||
| observed["payload"] = payload | ||
| return { | ||
| "choices": [ | ||
| { | ||
| "message": { | ||
| "content": ( | ||
| '{"answer_text": "The follow-up names the same bid.",' | ||
| ' "cited_source_numbers": [1]}' | ||
| ) | ||
| } | ||
| } | ||
| ] | ||
| } | ||
|
|
||
| monkeypatch.setattr(post_chat, "post_json", fake_post_json) | ||
| client = ContextualOrchestratorPostChatClient( | ||
| "https://orchestrator.example.test", "inference_token" | ||
| ) | ||
| answer = client.answer( | ||
| "What happened between these events?", | ||
| [ | ||
| ChatSourceDocument( | ||
| post_id="post-bid-follow-up", | ||
| post_title="Bid follow-up", | ||
| post_body="Northridge asked to confirm the bid date.", | ||
| ) | ||
| ], | ||
| ) | ||
|
|
||
| assert answer.cited_post_ids == ("post-bid-follow-up",) | ||
| assert observed["payload"]["mode"] == "verify" | ||
|
|
||
|
|
||
| def test_adjudication_requests_verify_mode(monkeypatch) -> None: | ||
| """Lineage adjudication must send verify on the wire, not a source substring.""" | ||
|
|
||
| observed: dict[str, object] = {} | ||
|
|
||
| def fake_post_json(url, payload, *, headers, timeout): | ||
| observed["payload"] = payload | ||
| return {"choices": [{"message": {"content": "0.91"}}]} | ||
|
|
||
| monkeypatch.setattr(adjudication_client, "post_json", fake_post_json) | ||
| client = adjudication_client.ContextualOrchestratorAdjudicationClient( | ||
| "https://orchestrator.example.test", "inference_token" | ||
| ) | ||
| confidence = client.judge( | ||
| "Quarterly budget review meeting notes", | ||
| "Budget review follow-up: revised quarterly numbers", | ||
| ) | ||
|
|
||
| assert confidence == 0.91 | ||
| assert observed["payload"]["mode"] == "verify" | ||
|
|
||
|
|
||
| def test_runtime_clients_do_not_force_single_model_route() -> None: | ||
| package_root = Path(__file__).resolve().parents[1] / "lineageweave" | ||
| violations: list[str] = [] | ||
| for path in sorted(package_root.glob("*.py")): | ||
| text = path.read_text(encoding="utf-8") | ||
| if '"mode": "route"' in text or "'mode': 'route'" in text: | ||
| violations.append(f"{path.name}: request payload") | ||
| if 'mode="route"' in text or "mode='route'" in text: | ||
| violations.append(f"{path.name}: constructor/call default") | ||
| if 'mode: str = "route"' in text or "mode: str = 'route'" in text: | ||
| violations.append(f"{path.name}: typed default") | ||
| assert violations == [] |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| """Contract tests for adaptive contextual-orchestrator consumer defaults.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| from pathlib import Path | ||
| import unittest | ||
|
|
||
| ROOT = Path(__file__).resolve().parents[1] | ||
| AUTO_CLIENTS = ( | ||
| "lineageweave/post_summary.py", | ||
| "lineageweave/post_evaluation.py", | ||
| "lineageweave/keyman_extraction.py", | ||
| "lineageweave/commitment_extraction.py", | ||
| "lineageweave/entity_relationship_classification.py", | ||
| ) | ||
| VERIFY_CLIENTS = ( | ||
| "lineageweave/post_chat.py", | ||
| "lineageweave/adjudication_client.py", | ||
| ) | ||
| _ROUTE_MARKERS = ( | ||
| '"mode": "route"', | ||
| 'mode="route"', | ||
| 'mode: str = "route"', | ||
| ) | ||
| _PAYLOAD_AUTO = '"mode": "auto"' | ||
| _PAYLOAD_VERIFY = '"mode": "verify"' | ||
| _TYPED_AUTO_DEFAULT = 'mode: str = "auto"' | ||
| _FORWARDED_MODE = '"mode": mode' | ||
|
|
||
|
|
||
| def _source(relative: str) -> str: | ||
| return (ROOT / relative).read_text(encoding="utf-8") | ||
|
|
||
|
|
||
| class AdaptiveOrchestratorDefaultTest(unittest.TestCase): | ||
| """Protect production clients from regressing to forced one-model routing.""" | ||
|
|
||
| def test_auto_clients_request_auto_and_never_force_route(self) -> None: | ||
| for relative in AUTO_CLIENTS: | ||
| source = _source(relative) | ||
| with self.subTest(path=relative): | ||
| for marker in _ROUTE_MARKERS: | ||
| self.assertNotIn(marker, source) | ||
| if relative.endswith("post_evaluation.py"): | ||
| self.assertIn(_FORWARDED_MODE, source) | ||
| self.assertIn(_TYPED_AUTO_DEFAULT, source) | ||
| continue | ||
| self.assertIn( | ||
| _PAYLOAD_AUTO, | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Whole-file |
||
| source, | ||
| f"{relative} must send a payload-level mode=auto literal", | ||
| ) | ||
|
|
||
| def test_verify_clients_keep_checked_judgment_and_never_force_route(self) -> None: | ||
| for relative in VERIFY_CLIENTS: | ||
| source = _source(relative) | ||
| with self.subTest(path=relative): | ||
| for marker in _ROUTE_MARKERS: | ||
| self.assertNotIn(marker, source) | ||
| self.assertIn( | ||
| _PAYLOAD_VERIFY, | ||
| source, | ||
| f"{relative} must send a payload-level mode=verify literal", | ||
| ) | ||
| self.assertNotIn( | ||
| _PAYLOAD_AUTO, | ||
| source, | ||
| f"{relative} must send verify, not a payload-level auto default", | ||
| ) | ||
|
|
||
| def test_docstring_mode_mention_is_not_a_payload_literal(self) -> None: | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This unit asserts that |
||
| """A class docstring contrast must not satisfy the auto/verify scan.""" | ||
|
|
||
| source = ( | ||
| '"""Calls the orchestrator with mode="auto", not mode="verify"."""\n' | ||
| "body = {\"messages\": [], \"mode\": \"route\"}\n" | ||
| ) | ||
| self.assertNotIn(_PAYLOAD_AUTO, source) | ||
| self.assertNotIn(_PAYLOAD_VERIFY, source) | ||
| self.assertIn('mode="auto"', source) | ||
| self.assertIn('mode="verify"', source) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| unittest.main() | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
AUTO_CLIENTSomitslineageweave/image_content.py.orchestrator_vision_clientPOSTs/v1/chat/completionsand is an ordinary ADR-0013 consumer. Add it as a factory-auto case (mode="auto"on the factory, written onto the body) rather than stuffingmodeinto genericOpenAiCompatibleVisionClient— that path must keep omittingmodeso OpenAI-compatible gateways do not 400.