From 2ceeaeac3311b54fbb624861db6bf726b9526130 Mon Sep 17 00:00:00 2001
From: Seongho Bae <me@seonghobae.me>
Date: Wed, 12 Aug 2026 05:07:14 +0900
Subject: [PATCH] fix(judge): reject ambiguous JSON responses

Combine nesting-depth bounds with duplicate-key rejection and exact
mode-specific top-level schema requirements so untrusted judge payloads
cannot smuggle alternate scores or unknown fields.
---
 CHANGELOG.md                    |  1 +
 python/fast_mlsirm/llm_judge.py | 35 ++++++++++++++++++++++++++++-----
 tests/test_llm_judge.py         | 32 ++++++++++++++++++++++++++++++
 3 files changed, 63 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 501150a1f..2ae818e94 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 ### Fixed
 
+- Reject ambiguous LLM-judge JSON with duplicate keys or unexpected top-level fields; require the exact mode-specific schema including advisory `accepted`.
 - Require compiled Rust ownership for public `s_x2` and `person_fit`, including prior-mean S-X² dispatch, with fail-closed errors when the core is missing.
 - Validate parallel-analysis integer controls and bound random-eigenvalue workspace before Rust dispatch.
 - Cap LLM-judge response JSON nesting at 32 levels before parse to prevent recursive-object resource exhaustion.
diff --git a/python/fast_mlsirm/llm_judge.py b/python/fast_mlsirm/llm_judge.py
index 57f48267b..48314549f 100644
--- a/python/fast_mlsirm/llm_judge.py
+++ b/python/fast_mlsirm/llm_judge.py
@@ -27,6 +27,19 @@ class JudgeFormatError(ValueError):
     """Raised when a judge response is not a bounded, interpretable decision."""
 
 
+class _DuplicateJsonKeyError(ValueError):
+    """Internal signal for duplicate JSON object members."""
+
+
+def _duplicate_free_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    value: dict[str, Any] = {}
+    for key, member in pairs:
+        if key in value:
+            raise _DuplicateJsonKeyError(key)
+        value[key] = member
+    return value
+
+
 def _category_count(value: Any) -> int:
     if (
         not isinstance(value, int)
@@ -296,15 +309,21 @@ def _validate_raw_json_depth(content: str) -> None:
             depth -= 1
 
 
-def _response_object(raw: str) -> dict[str, Any]:
+def _response_object(raw: str, *, required_fields: set[str]) -> dict[str, Any]:
     text = raw.strip()
     _validate_raw_json_depth(text)
     try:
-        value = json.loads(text)
+        value = json.loads(text, object_pairs_hook=_duplicate_free_object)
+    except _DuplicateJsonKeyError as exc:
+        raise JudgeFormatError("judge response contains duplicate JSON object keys") from exc
     except json.JSONDecodeError as exc:
         raise JudgeFormatError("judge response JSON is invalid") from exc
     if not isinstance(value, dict):
         raise JudgeFormatError("judge response must be a JSON object")
+    if set(value) != required_fields:
+        raise JudgeFormatError(
+            "judge response must contain exactly the required fields"
+        )
     return value
 
 
@@ -428,10 +447,16 @@ def judge(
             raw = _bounded_text(completion.get("answer"), "judge answer")
         except ValueError as exc:
             raise JudgeFormatError(str(exc)) from exc
-        parsed = _response_object(raw)
+        criterion_field = (
+            "criterion_categories" if category_count is not None else "criterion_scores"
+        )
+        parsed = _response_object(
+            raw,
+            required_fields={"score", "accepted", "rationale", criterion_field},
+        )
         advisory_accepted = parsed.get("accepted")
-        if advisory_accepted is not None and not isinstance(advisory_accepted, bool):
-            raise JudgeFormatError("accepted must be a boolean when present")
+        if not isinstance(advisory_accepted, bool):
+            raise JudgeFormatError("accepted must be a boolean")
         try:
             rationale = _bounded_text(parsed.get("rationale"), "rationale")
         except ValueError as exc:
diff --git a/tests/test_llm_judge.py b/tests/test_llm_judge.py
index 55df0583d..470e04e9e 100644
--- a/tests/test_llm_judge.py
+++ b/tests/test_llm_judge.py
@@ -108,6 +108,38 @@ def test_judge_rejects_wrapped_or_fenced_json() -> None:
             )
 
 
+
+def test_judge_rejects_duplicate_and_unknown_top_level_fields() -> None:
+    duplicate = (
+        '{"score":0.8,"accepted":true,"rationale":"supported",'
+        '"criterion_scores":{"task_alignment":0.8,"factual_support":0.8},'
+        '"score":0.2}'
+    )
+    unknown = json.loads(_payload())
+    unknown["unexpected"] = "ignored fields are unsafe"
+    for answer in (duplicate, json.dumps(unknown)):
+        with pytest.raises(JudgeFormatError, match="exactly|duplicate"):
+            ContextualOrchestratorJudge(_FakeOrchestrator(answer)).judge(
+                task="task",
+                answer="answer",
+                criteria=CRITERIA,
+            )
+
+
+def test_judge_rejects_duplicate_nested_criterion_fields() -> None:
+    answer = (
+        '{"score":0.8,"accepted":true,"rationale":"supported",'
+        '"criterion_scores":{"task_alignment":0.8,"task_alignment":0.2,'
+        '"factual_support":0.8}}'
+    )
+    with pytest.raises(JudgeFormatError, match="duplicate"):
+        ContextualOrchestratorJudge(_FakeOrchestrator(answer)).judge(
+            task="task",
+            answer="answer",
+            criteria=CRITERIA,
+        )
+
+
 def test_judge_result_projects_only_multiple_criteria_to_irt_items() -> None:
     result = ContextualOrchestratorJudge(_FakeOrchestrator(_payload())).judge(
         task="task",