diff --git a/docs/adr/0100-adaptive-contextual-orchestrator-default.md b/docs/adr/0100-adaptive-contextual-orchestrator-default.md new file mode 100644 index 000000000..f15c69f5e --- /dev/null +++ b/docs/adr/0100-adaptive-contextual-orchestrator-default.md @@ -0,0 +1,36 @@ +# ADR-0100: Adaptive contextual-orchestrator mode is the LLM-judge default + +Status: **Accepted** +Date: 2026-08-17 + +## Context + +`fast-mlsirm` delegates model transport and orchestration to an injected contextual-orchestrator adapter, while keeping rubric validation, result parsing, and psychometric projection inside this repository. The adapter previously defaulted ordinary judge calls to fixed `route` mode, which made the consumer choose execution topology even though contextual-orchestrator owns routing, verification, fallback, and cost policy. + +The cross-repository boundary is already version-marked by `contextual-orchestrator-contract-v1`. Current live provider responses do not expose a separate immutable request/result artifact digest or mandatory provider/model identity fields, so this consumer must not invent those fields and reject otherwise valid live responses. A stronger result-schema version must first be published by the owning contextual-orchestrator contract and then adopted here fail-closed. + +## Decision + +`ContextualOrchestratorJudge` defaults to `mode="auto"`. + +`auto` delegates execution topology to contextual-orchestrator. The orchestration plane may keep a simple task on one route or allocate deeper verification when its policy determines that additional reliability is warranted, while applying its own known-cost policy. Explicit `route` and `conduct` remain available for controlled ablation, incident response, and documented operational requirements. + +Construction remains fail-closed on the public `contextual-orchestrator-contract-v1` marker. The caller continues to validate bounded model output and to preserve the returned validated orchestration mode, trace count, and usage evidence. `fast-mlsirm` does not infer provider/model provenance that the live upstream contract does not publish. When contextual-orchestrator publishes a stronger versioned request/result schema or immutable artifact digest, this ADR requires a compatibility update here before relying on that new evidence surface. + +## Consequences + +- Ordinary consumers no longer pin judge calls to fixed single-route execution. +- Explicit `route` and `conduct` behavior remains testable and observable. +- The adapter remains provider-neutral and keeps model output untrusted. +- Cross-repository compatibility remains fail-closed at the strongest contract actually published by the owning repository; no consumer-only response fields are fabricated. +- No psychometric numerical arithmetic moves out of Rust-owned production kernels. + +## Evidence + +Omidvar, H., & Akhlaghi, V. (2026). *A communication-theoretic framework for LLM agents: Cost-aware adaptive reliability* [Preprint]. arXiv. https://doi.org/10.48550/arXiv.2605.09121 + +The study frames difficulty-adaptive routing as a reliability operator and reports a cost-aware semantic router that traverses an empirical quality-cost frontier, supporting per-task allocation rather than one fixed model-technique-budget choice. + +Tang, Y., Cetin, E., Xu, J., Sun, Q., Nielsen, S., Richard, V., Goda, H., Tymchenko, I., Nguyen, N., Lee, H., Ashiga, M., Kotyan, S., Kuroki, S., & Clanuwat, T. (2026). *Sakana Fugu technical report* [Technical report]. arXiv. https://doi.org/10.48550/arXiv.2606.21228 + +The report describes query-adaptive agentic scaffolds and distinct latency-balanced versus quality-prioritized operating points, supporting adaptive orchestration instead of one fixed topology. diff --git a/docs/changelog.d/953-adaptive-judge-default.md b/docs/changelog.d/953-adaptive-judge-default.md new file mode 100644 index 000000000..c841a852c --- /dev/null +++ b/docs/changelog.d/953-adaptive-judge-default.md @@ -0,0 +1,5 @@ +# Default LLM judge orchestration to adaptive auto mode + +## Changed + +- `ContextualOrchestratorJudge` now defaults ordinary calls to contextual-orchestrator `auto` mode while preserving explicit `route` and `conduct` overrides and the fail-closed `contextual-orchestrator-contract-v1` adapter boundary. diff --git a/python/fast_mlsirm/llm_judge.py b/python/fast_mlsirm/llm_judge.py index d66ab7394..27f957ef5 100644 --- a/python/fast_mlsirm/llm_judge.py +++ b/python/fast_mlsirm/llm_judge.py @@ -444,9 +444,9 @@ def _single_call_failure_evidence( class ContextualOrchestratorJudge: - """Evaluate one answer through a marked contextual-orchestrator adapter.""" + """Evaluate through a marked contextual-orchestrator adapter using adaptive routing by default.""" - def __init__(self, orchestrator: Any, *, mode: str = "route", accept_threshold: float = 0.7) -> None: + def __init__(self, orchestrator: Any, *, mode: str = "auto", accept_threshold: float = 0.7) -> None: if not callable(getattr(orchestrator, "complete", None)): raise TypeError("orchestrator must provide complete(messages, mode=...)") try: @@ -658,9 +658,6 @@ def judge_boundary(request: tuple[JudgeCriterion, int]) -> dict[str, Any]: ) if type(parsed["meets_threshold"]) is not bool: raise JudgeFormatError("meets_threshold must be a boolean") - # Keep the parsed ordinal signal in bounded failure evidence so - # a non-monotone comparison can be audited without retaining - # the full model response. record["meets_threshold"] = parsed["meets_threshold"] rationale = _bounded_text(parsed["rationale"], "rationale") record["parse_status"] = "passed" @@ -731,8 +728,6 @@ def failure_evidence( if max_workers == 1: outcomes = [judge_boundary(request) for request in requests] else: - # Reuse contextual-orchestrator's already-bounded local setting; - # generic injected orchestrators remain sequential by default. with ThreadPoolExecutor(max_workers=max_workers) as pool: outcomes = list(pool.map(judge_boundary, requests)) @@ -811,8 +806,6 @@ def judge( normalized_criteria, category_count ) if category_method is None: - # K-way category selection is vulnerable to score-option effects; - # use independent Boolean boundaries for implicit polytomous calls. category_method = "binary_threshold" if category_count is not None else "direct" elif ( type(category_method) is not str @@ -832,7 +825,7 @@ def judge( criterion_payload = [criterion.to_dict() for criterion in normalized_criteria] reference_block = reference_answer or "(none supplied)" if category_method == "binary_threshold": - assert category_count is not None # validated above + assert category_count is not None call_count = len(normalized_criteria) * (category_count - 1) if call_count > MAX_BINARY_THRESHOLD_CALLS: raise ValueError( @@ -1063,6 +1056,7 @@ def judge( error_type=type(exc).__name__, ), ) from None + def response_failure(exc: JudgeFormatError) -> JudgeFormatError: return JudgeFormatError( str(exc), @@ -1086,8 +1080,6 @@ def response_failure(exc: JudgeFormatError) -> JudgeFormatError: expected_id_set = set(expected_ids) criterion_categories: dict[str, int] | None = None if category_count is not None: - # Validate the redundant field's shape, but derive the accepted score - # from the ordered category items below rather than trusting it. _score(parsed.get("score"), "score") criterion_categories = {} if category_method == "cumulative_threshold": diff --git a/tests/test_llm_judge.py b/tests/test_llm_judge.py index 24dd07aa6..6df59fbc4 100644 --- a/tests/test_llm_judge.py +++ b/tests/test_llm_judge.py @@ -176,7 +176,7 @@ def test_judge_uses_contextual_orchestrator_route_and_reports_usage() -> None: assert result.score == 0.8 assert result.trace_step_count == 1 assert dict(result.usage) == {"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 12} - assert orchestrator.calls[0][1] == "route" + assert orchestrator.calls[0][1] == "auto" prompt = orchestrator.calls[0][0][1]["content"] payload = json.loads(prompt.split("\n", 1)[1]) assert payload["task"] == "Explain the release plan." diff --git a/tests/test_llm_judge_modes.py b/tests/test_llm_judge_modes.py new file mode 100644 index 000000000..8aaf094b1 --- /dev/null +++ b/tests/test_llm_judge_modes.py @@ -0,0 +1,77 @@ +"""Preserve contextual-orchestrator mode defaults and explicit overrides.""" + +from __future__ import annotations + +import json + +import pytest + +from fast_mlsirm.llm_judge import ( + CONTEXTUAL_ORCHESTRATOR_CONTRACT_V1, + ContextualOrchestratorJudge, + JudgeCriterion, +) + + +class _RecordingOrchestrator: + """Return one valid decision while recording the requested mode.""" + + contextual_orchestrator_contract = CONTEXTUAL_ORCHESTRATOR_CONTRACT_V1 + + def __init__(self) -> None: + self.modes: list[str] = [] + + def complete( + self, + messages: list[dict[str, str]], + *, + mode: str, + ) -> dict[str, object]: + """Record ``mode`` and return a bounded contract-compatible fixture.""" + assert messages + self.modes.append(mode) + return { + "mode": mode, + "answer": json.dumps( + { + "score": 1.0, + "accepted": True, + "rationale": "The answer satisfies the criterion.", + "criterion_scores": {"task_alignment": 1.0}, + } + ), + "trace": [], + } + + +def _judge(mode: str | None = None) -> tuple[_RecordingOrchestrator, object]: + """Execute one judge call with either the default or an explicit mode.""" + orchestrator = _RecordingOrchestrator() + judge = ( + ContextualOrchestratorJudge(orchestrator) + if mode is None + else ContextualOrchestratorJudge(orchestrator, mode=mode) + ) + result = judge.judge( + task="Assess the release plan.", + answer="Use a staged release with rollback.", + criteria=[JudgeCriterion("task_alignment", "The answer addresses the task.")], + ) + return orchestrator, result + + +def test_judge_defaults_to_adaptive_auto_mode() -> None: + """Ordinary consumers delegate execution topology to contextual-orchestrator.""" + orchestrator, result = _judge() + + assert orchestrator.modes == ["auto"] + assert result.orchestration_mode == "auto" + + +@pytest.mark.parametrize("mode", ["route", "conduct"]) +def test_explicit_judge_mode_reaches_contextual_orchestrator(mode: str) -> None: + """Explicit route/conduct overrides remain observable for controlled use.""" + orchestrator, result = _judge(mode) + + assert orchestrator.modes == [mode] + assert result.orchestration_mode == mode