diff --git a/README.md b/README.md index 65f57dd4c..946da28b2 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Non-mock providers must use `https://` URLs and a **resolvable KV credential** One public interface: -- `/v1/chat/completions` accepts normal chat messages, and `"stream": true` returns an OpenAI-compatible `text/event-stream` of `chat.completion.chunk` deltas terminated by `data: [DONE]`. In **route** mode the worker's tokens are streamed live as they arrive from the provider (real token streaming); in **conduct** mode the multi-step answer is produced then framed as deltas (a workflow can't honestly token-stream a synthesizer that hasn't run yet). +- `/v1/chat/completions` accepts normal chat messages, and `"stream": true` returns an OpenAI-compatible `text/event-stream` of `chat.completion.chunk` deltas terminated by `data: [DONE]`. In **route** mode the worker's tokens are streamed live as they arrive from the provider (real token streaming); in **conduct** mode the multi-step answer is produced then framed as deltas (a workflow can't honestly token-stream a synthesizer that hasn't run yet). Send one of `orchestration` / `orchestration_mode` / `mode`, or omit them — mixed `orchestration=route` plus `mode=conduct` is `invalid_mode`. JSON `null` and `""` are omit-equivalent; do not send whitespace-only mode. - `TaskOrchestrator.complete()` decides whether to route to one worker or run a short workflow. - `TaskOrchestrator.compare_to_baseline(prompts, mode)` (CLI `--eval PROMPT...`) measures the orchestration engine against a single-worker baseline — per-prompt and aggregate latency plus a structural coverage delta (contributing steps + verifier-pass presence). It is a measured tradeoff report, not a human-quality claim. - Responses include orchestration mode metadata, and trusted callers can request the full trace for audit. @@ -256,6 +256,7 @@ python tests/test_admin_contract.py python tests/test_conventions.py python tests/test_api_contract.py python tests/test_security_hardening.py +python tests/test_chat_orchestration_mode_http_honesty.py python tests/test_repository_security_metadata.py python tests/test_product_planning_contract.py python tests/test_plugin_driven_artifacts.py diff --git a/conductor/tracks.md b/conductor/tracks.md index 968c08ef8..152b894b3 100644 --- a/conductor/tracks.md +++ b/conductor/tracks.md @@ -4,3 +4,4 @@ |---|---|---| | 001-paper-grounded-orchestrator | active | Implement the source-backed orchestration contract with TDD, DDD, and CDD | | 002-enterprise-design-foundation | active | Add paper-grounded screen design, user stories, REST API, code/DB conventions, and i18n | +| 003-mode-alias-honesty | active | Fail closed when `orchestration` / `orchestration_mode` / `mode` disagree so a Conductor workflow cannot hide behind a Fugu route | diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index c58d4cb79..aadc256f9 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -183,6 +183,47 @@ def _validate_mode(mode: Any) -> str: return mode +def _resolve_requested_chat_mode(body: dict[str, Any]) -> str: + """Resolve ``orchestration`` / ``orchestration_mode`` / ``mode`` without first-wins hide. + + Each present alias is validated on its own. JSON ``null`` and ``""`` are + omit-equivalent so SDK optional defaults stay no-ops. Whitespace-only + values fail closed through ``_validate_mode``. Distinct non-omit aliases + fail closed so ``orchestration=route`` cannot hide ``mode=conduct`` and + bill a Fugu-style single-worker route for a Conductor workflow the buyer + asked for (Nielsen et al., 2025; Xu et al., 2025). When every alias is + omitted, the result is ``auto``. + + Args: + body: Chat Completions JSON object after unknown-key rejection. + + Returns: + One of ``auto``, ``route``, or ``conduct``. + + Raises: + RequestError: invalid, whitespace-only, or disagreeing aliases. + """ + resolved_modes: list[str] = [] + for key in ("orchestration", "orchestration_mode", "mode"): + if key not in body: + continue + raw_mode = body.get(key) + if raw_mode is None or raw_mode == "": + continue + resolved_modes.append(_validate_mode(raw_mode)) + if not resolved_modes: + return "auto" + unique_modes = set(resolved_modes) + if len(unique_modes) > 1: + raise RequestError( + 400, + "invalid_mode", + "orchestration, orchestration_mode, and mode must agree; " + "omit unused aliases or send the same value on each", + ) + return resolved_modes[0] + + def _validate_messages(messages: Any) -> list[dict[str, str]]: if not isinstance(messages, list) or not messages: raise RequestError(400, "invalid_message", "messages must be a non-empty array") @@ -732,7 +773,7 @@ def do_POST(self) -> None: # noqa: N802 self._send(proxied) return messages = _validate_messages(body.get("messages")) - mode = _validate_mode(body.get("orchestration") or body.get("orchestration_mode") or body.get("mode") or "auto") + mode = _resolve_requested_chat_mode(body) include_trace = bool(body.get("include_orchestration_trace", security.expose_trace_by_default)) stream = body.get("stream", False) if not isinstance(stream, bool): diff --git a/docs/architecture.md b/docs/architecture.md index c0f63a81e..eaa652c2d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -4,8 +4,8 @@ - Sakana AI launch article, "Sakana Fugu: One Model to Command Them All" (June 22, 2026): https://sakana.ai/fugu-release/ - Sakana Fugu Technical Report: https://github.com/SakanaAI/fugu/blob/main/Fugu_technical_report.pdf -- TRINITY: An Evolved LLM Coordinator: https://arxiv.org/abs/2512.04695 -- Learning to Orchestrate Agents in Natural Language with the Conductor: https://arxiv.org/abs/2512.04388 +- Xu, J., Sun, Q., Schwendeman, P., Nielsen, S., Cetin, E., & Tang, Y. (2025). *Trinity: An evolved LLM coordinator*. arXiv. https://doi.org/10.48550/arXiv.2512.04695 +- Nielsen, S., Cetin, E., Schwendeman, P., Sun, Q., Xu, J., & Tang, Y. (2025). *Learning to orchestrate agents in natural language with the Conductor*. arXiv. https://doi.org/10.48550/arXiv.2512.04388 ## What The Architecture Is @@ -26,6 +26,7 @@ The Fugu report combines these ideas into production constraints: - Fugu-Ultra is optimized for quality by generating deeper workflows over a broader agent pool. - The agent pool is swappable, allowing provider preference, model exclusion, and compliance controls. - Multi-agent tool/function-call workflows need memory discipline: isolate agents inside the current workflow, but keep useful shared memory across turns. +- Chat Completions mode aliases (`orchestration`, `orchestration_mode`, `mode`) are checked on their own. Mixed `orchestration=route` plus `mode=conduct` fails closed so a Conductor workflow the buyer asked for cannot hide behind a Fugu-style single-worker route (Nielsen et al., 2025; Xu et al., 2025). JSON `null` and `""` stay omit-equivalent; whitespace-only mode is `invalid_mode`. ## Implementation Mapping diff --git a/docs/fuzzing.md b/docs/fuzzing.md index 9897b2bd2..dc20df395 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -19,9 +19,10 @@ The surfaces were located with CodeGraph (`codegraph explore "parse decode deserialize request config validate untrusted input"`): 1. **HTTP request body** — `server._coerce_json` / `_reject_unknown_keys` / - `_validate_mode` / `_validate_messages`. Arbitrary bytes must normalise to a - validated structure or raise `RequestError` / a JSON decode error — never an - unhandled crash. + `_validate_mode` / `_resolve_requested_chat_mode` / `_validate_messages`. + Arbitrary bytes must normalise to a validated structure or raise + `RequestError` / a JSON decode error — never an unhandled crash. Mixed + `orchestration` / `mode` aliases must agree or fail closed. 2. **Agent config** — `orchestrator.ModelAgent.from_dict`. Arbitrary decoded JSON must yield a well-typed `ModelAgent` or raise `KeyError`/`TypeError`/ `ValueError`. diff --git a/docs/papers/README.md b/docs/papers/README.md index 65a89d2af..1ac55b6ea 100644 --- a/docs/papers/README.md +++ b/docs/papers/README.md @@ -43,6 +43,26 @@ motivate throughput-oriented **batched** inference and the load-balancing that makes the latency-tolerant batch route economical. Those sources are referenced but not vendored here so this repository remains one deployable control plane. +## Mode-alias honesty (route vs conduct) + +Buyers who send `orchestration=route` plus `mode=conduct` asked for a Conductor +workflow (Nielsen et al., 2025) with TRINITY-style role traces (Xu et al., 2025). +A first-wins `or` chain billed a Fugu-style single-worker route instead. Each of +`orchestration` / `orchestration_mode` / `mode` is checked on its own; disagreeing +aliases fail closed before a `chat.completion` is billed. PDFs are not vendored +here (redistribution not confirmed); cite + link + summary only. + +- Nielsen, S., Cetin, E., Schwendeman, P., Sun, Q., Xu, J., & Tang, Y. (2025). + *Learning to orchestrate agents in natural language with the Conductor*. + arXiv. https://doi.org/10.48550/arXiv.2512.04388 + Grounds natural-language workflow steps, assigned workers, and access lists. + Mixed aliases must not hide a workflow the buyer asked for. +- Xu, J., Sun, Q., Schwendeman, P., Nielsen, S., Cetin, E., & Tang, Y. (2025). + *Trinity: An evolved LLM coordinator*. arXiv. + https://doi.org/10.48550/arXiv.2512.04695 + Grounds thinker / worker / verifier role traces. A silent route completion + drops those roles. + > Citations are provided for scholarly attribution. Redistribution here relies > on the arXiv non-exclusive distribution license each author granted; no > GPL/AGPL-licensed material is vendored anywhere in this repository. diff --git a/docs/rest_api_design.md b/docs/rest_api_design.md index 9378e5a37..c4a44416b 100644 --- a/docs/rest_api_design.md +++ b/docs/rest_api_design.md @@ -8,6 +8,11 @@ - Error shape: `{"error_code": "...", "error_message": "...", "error_detail": {...}}` in production. - Pagination shape: `items`, `total_count`, `page_number`, `page_size` for collections. - OpenAI-compatible compatibility endpoint remains `/v1/chat/completions`. +- On `/v1/chat/completions`, send one of `orchestration` / `orchestration_mode` / + `mode`, or omit them. Mixed `orchestration=route` plus `mode=conduct` is + `invalid_mode` — aliases must agree. JSON `null` and `""` are omit-equivalent; + do not send whitespace-only mode. Next action: pick `auto`, `route`, or + `conduct` on one key and omit the others. ## Current Endpoints diff --git a/fuzz/targets.py b/fuzz/targets.py index d0c344462..a167e3cfc 100644 --- a/fuzz/targets.py +++ b/fuzz/targets.py @@ -11,8 +11,9 @@ CodeGraph (``codegraph explore``) surfaced these four surfaces as the ones that consume untrusted bytes/JSON: -1. ``server._coerce_json`` / ``_validate_mode`` / ``_validate_messages`` / - ``_reject_unknown_keys`` -- the HTTP request-body parser and validators. +1. ``server._coerce_json`` / ``_validate_mode`` / ``_resolve_requested_chat_mode`` / + ``_validate_messages`` / ``_reject_unknown_keys`` -- the HTTP request-body + parser and validators. Mixed mode aliases must agree or fail closed. 2. ``orchestrator.ModelAgent.from_dict`` -- the agent-pool config parser. 3. ``orchestrator.redact_text`` / ``redact_value`` -- secret/PII redaction run over arbitrary trace payloads (regex + recursion). @@ -93,6 +94,15 @@ def exercise_request_body(raw: bytes) -> None: else: assert mode in server.ALLOWED_MODES + # Per-key alias resolution: mixed orchestration/mode values must agree or + # raise RequestError — never pick the first truthy alias. + try: + resolved_mode = server._resolve_requested_chat_mode(body) + except RequestError: + pass + else: + assert resolved_mode in server.ALLOWED_MODES + # Message validation: returns a normalised list or raises RequestError. if "messages" in body: try: diff --git a/tests/test_chat_orchestration_mode_http_honesty.py b/tests/test_chat_orchestration_mode_http_honesty.py new file mode 100644 index 000000000..99dd7b6e4 --- /dev/null +++ b/tests/test_chat_orchestration_mode_http_honesty.py @@ -0,0 +1,191 @@ +"""Live HTTP: mixed mode aliases must not hide a Conductor workflow. + +A buyer who sends ``orchestration=route`` plus ``mode=conduct`` asked for a +Conductor workflow (Nielsen et al., 2025). The first-wins ``or`` chain billed a +Fugu-style single-worker route instead. Each alias is checked on its own. +""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_orchestration_mode_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ + ModelAgent("planner_agent", "mock-planner", tags=("planning", "reasoning")), + ModelAgent("builder_agent", "mock-builder", tags=("coding", "writing")), + ModelAgent("reviewer_agent", "mock-reviewer", tags=("verification", "review")), + ] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server() -> tuple[object, threading.Thread, int]: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_mixed_route_and_conduct_aliases() -> None: + """``orchestration=route`` must not hide ``mode=conduct`` on the chat path.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "analyze, implement, and verify the invoice parser"}], + "orchestration": "route", + "mode": "conduct", + }, + ) + assert status == 400, body + assert body["error"]["code"] == "invalid_mode" + assert "agree" in body["error"]["message"] + assert "choices" not in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_mixed_route_and_whitespace_mode() -> None: + """``orchestration=route`` must not hide whitespace-only ``mode``.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "say hi"}], + "orchestration": "route", + "mode": " ", + }, + ) + assert status == 400, body + assert body["error"]["code"] == "invalid_mode" + assert "choices" not in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_agreeing_route_aliases() -> None: + """The same value on two aliases is an honest no-op, not a conflict.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "say hi"}], + "orchestration": "route", + "mode": "route", + }, + ) + assert status == 200, body + assert body["orchestration"]["mode"] == "route" + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_string_mode_as_omit() -> None: + """JSON empty-string mode stays omit-equivalent; spaces do not.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "say hi"}], + "orchestration": "route", + "mode": "", + }, + ) + assert status == 200, body + assert body["orchestration"]["mode"] == "route" + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_null_mode_as_omit() -> None: + """JSON null mode stays omit-equivalent.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "say hi"}], + "mode": None, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_mode_conduct() -> None: + """A single ``mode=conduct`` still runs the Conductor workflow.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "analyze, implement, and verify the invoice parser"}], + "mode": "conduct", + }, + ) + assert status == 200, body + assert body["orchestration"]["mode"] == "conduct" + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_mixed_route_and_conduct_aliases() + test_http_chat_rejects_mixed_route_and_whitespace_mode() + test_http_chat_accepts_agreeing_route_aliases() + test_http_chat_accepts_empty_string_mode_as_omit() + test_http_chat_accepts_null_mode_as_omit() + test_http_chat_accepts_mode_conduct() + print("ok")