From b58d65e89900f9aa95e943b7338b9663f13661a6 Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Sat, 1 Aug 2026 12:25:23 -0500 Subject: [PATCH 1/2] fix(verifier): normalize structured provider content --- scripts/langchain/pr_verifier.py | 24 +++++++++++++++---- .../test_pr_verifier_structured_output.py | 24 +++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/scripts/langchain/pr_verifier.py b/scripts/langchain/pr_verifier.py index 0a127ea7b..1a40b457e 100755 --- a/scripts/langchain/pr_verifier.py +++ b/scripts/langchain/pr_verifier.py @@ -675,12 +675,28 @@ def _fallback_evaluation( ) +def _coerce_response_content(content: object) -> str: + """Return text from provider response blocks without losing a safe fallback.""" + if isinstance(content, str): + return content + if isinstance(content, list): + text_blocks = [ + block["text"] + for block in content + if isinstance(block, dict) and isinstance(block.get("text"), str) + ] + if text_blocks: + return "\n".join(text_blocks) + return json.dumps(content, default=str) + + def _parse_llm_response( - content: str, provider: str, *, client: object | None = None + content: object, provider: str, *, client: object | None = None ) -> EvaluationResult: + content_text = _coerce_response_content(content) repair = _build_verifier_repair_callback(client) if client is not None else None parsed = parse_structured_output( - content, + content_text, EvaluationPayload, repair=repair, max_repair_attempts=SCHEMA_REPAIR_POLICY.max_attempts, @@ -704,7 +720,7 @@ def _parse_llm_response( summary=None, provider_used=provider, used_llm=True, - raw_content=content, + raw_content=content_text, error=error, ) @@ -717,7 +733,7 @@ def _parse_llm_response( summary=payload.summary, provider_used=provider, used_llm=True, - raw_content=parsed.raw_content or content, + raw_content=parsed.raw_content or content_text, ) diff --git a/tests/scripts/test_pr_verifier_structured_output.py b/tests/scripts/test_pr_verifier_structured_output.py index b6013a2df..a7e76124d 100644 --- a/tests/scripts/test_pr_verifier_structured_output.py +++ b/tests/scripts/test_pr_verifier_structured_output.py @@ -83,6 +83,30 @@ def test_comparison_runner_repairs_malformed_output() -> None: assert mock_client.invoke.call_count == 2 +def test_comparison_runner_handles_structured_response_content() -> None: + payload = json.dumps(_valid_payload()) + response = _response_with( + [ + {"type": "thinking", "thinking": "reviewing", "signature": "opaque"}, + {"type": "text", "text": payload}, + ] + ) + mock_client = mock.MagicMock() + mock_client.invoke.return_value = response + + runner = pr_verifier.ComparisonRunner( + context="context", + diff=None, + prompt="prompt", + clients=[(mock_client, "anthropic", "claude-sonnet")], + ) + + result = runner.run_single(mock_client, "anthropic", "claude-sonnet") + + assert result.verdict == "PASS" + assert result.raw_content == payload + + def test_evaluate_pr_valid_output_no_repair(monkeypatch: pytest.MonkeyPatch) -> None: payload = _valid_payload() good = json.dumps(payload) From c82373955b7d3dc8f36d4fd5152025dc73cf27a7 Mon Sep 17 00:00:00 2001 From: Cursor Automation Date: Sat, 1 Aug 2026 12:44:11 -0500 Subject: [PATCH 2/2] fix(verifier): preserve split text blocks and normalize repair responses Concatenate provider text blocks without a separator so a JSON document split across blocks reassembles faithfully, and coerce schema-repair responses through the same normalization so a structured repair reply is not rejected as CONCERNS. --- scripts/langchain/pr_verifier.py | 10 +++- .../test_pr_verifier_structured_output.py | 59 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/scripts/langchain/pr_verifier.py b/scripts/langchain/pr_verifier.py index 1a40b457e..93f02ca06 100755 --- a/scripts/langchain/pr_verifier.py +++ b/scripts/langchain/pr_verifier.py @@ -686,7 +686,10 @@ def _coerce_response_content(content: object) -> str: if isinstance(block, dict) and isinstance(block.get("text"), str) ] if text_blocks: - return "\n".join(text_blocks) + # Concatenate without a separator: a provider may split one JSON + # document across blocks, and an inserted newline inside a string + # literal would make the reassembled payload invalid JSON. + return "".join(text_blocks) return json.dumps(content, default=str) @@ -741,11 +744,14 @@ def _build_verifier_repair_callback(client: object) -> Callable[[str, str, str], repair = build_repair_callback(client) def _repair(schema_json: str, validation_errors: str, raw_response: str) -> str | None: - return repair( + repaired = repair( schema_json, validation_errors, _cap_prompt_text(raw_response, EVAL_SCHEMA_REPAIR_BUDGET_TOKENS), ) + if not repaired: + return None + return _coerce_response_content(repaired) return _repair diff --git a/tests/scripts/test_pr_verifier_structured_output.py b/tests/scripts/test_pr_verifier_structured_output.py index a7e76124d..c708d893d 100644 --- a/tests/scripts/test_pr_verifier_structured_output.py +++ b/tests/scripts/test_pr_verifier_structured_output.py @@ -107,6 +107,65 @@ def test_comparison_runner_handles_structured_response_content() -> None: assert result.raw_content == payload +def test_comparison_runner_concatenates_split_text_blocks() -> None: + payload = json.dumps(_valid_payload()) + # Split inside the summary string literal, where an inserted newline would + # be an unescaped control character and invalidate the JSON. + split = payload.index("Looks good.") + 5 + response = _response_with( + [ + {"type": "text", "text": payload[:split]}, + {"type": "text", "text": payload[split:]}, + ] + ) + mock_client = mock.MagicMock() + mock_client.invoke.return_value = response + + runner = pr_verifier.ComparisonRunner( + context="context", + diff=None, + prompt="prompt", + clients=[(mock_client, "anthropic", "claude-sonnet")], + ) + + result = runner.run_single(mock_client, "anthropic", "claude-sonnet") + + assert result.verdict == "PASS" + assert result.raw_content == payload + assert mock_client.invoke.call_count == 1 + + +def test_comparison_runner_normalizes_structured_repair_response() -> None: + payload = json.dumps(_valid_payload()) + malformed = _response_with( + [ + {"type": "thinking", "thinking": "reviewing", "signature": "opaque"}, + {"type": "text", "text": "```json\n" + payload + "\n```"}, + ] + ) + repaired = _response_with( + [ + {"type": "thinking", "thinking": "repairing", "signature": "opaque"}, + {"type": "text", "text": payload}, + ] + ) + mock_client = mock.MagicMock() + mock_client.invoke.side_effect = [malformed, repaired] + + runner = pr_verifier.ComparisonRunner( + context="context", + diff=None, + prompt="prompt", + clients=[(mock_client, "anthropic", "claude-sonnet")], + ) + + result = runner.run_single(mock_client, "anthropic", "claude-sonnet") + + assert result.verdict == "PASS" + assert result.raw_content == payload + assert mock_client.invoke.call_count == 2 + + def test_evaluate_pr_valid_output_no_repair(monkeypatch: pytest.MonkeyPatch) -> None: payload = _valid_payload() good = json.dumps(payload)