diff --git a/scripts/langchain/pr_verifier.py b/scripts/langchain/pr_verifier.py index 0a127ea7b..93f02ca06 100755 --- a/scripts/langchain/pr_verifier.py +++ b/scripts/langchain/pr_verifier.py @@ -675,12 +675,31 @@ def _fallback_evaluation( ) +def _coerce_response_content(content: object) -> str: + """Return text from provider response blocks without losing a safe fallback.""" + if isinstance(content, str): + return content + if isinstance(content, list): + text_blocks = [ + block["text"] + for block in content + if isinstance(block, dict) and isinstance(block.get("text"), str) + ] + if text_blocks: + # Concatenate without a separator: a provider may split one JSON + # document across blocks, and an inserted newline inside a string + # literal would make the reassembled payload invalid JSON. + return "".join(text_blocks) + return json.dumps(content, default=str) + + def _parse_llm_response( - content: str, provider: str, *, client: object | None = None + content: object, provider: str, *, client: object | None = None ) -> EvaluationResult: + content_text = _coerce_response_content(content) repair = _build_verifier_repair_callback(client) if client is not None else None parsed = parse_structured_output( - content, + content_text, EvaluationPayload, repair=repair, max_repair_attempts=SCHEMA_REPAIR_POLICY.max_attempts, @@ -704,7 +723,7 @@ def _parse_llm_response( summary=None, provider_used=provider, used_llm=True, - raw_content=content, + raw_content=content_text, error=error, ) @@ -717,7 +736,7 @@ def _parse_llm_response( summary=payload.summary, provider_used=provider, used_llm=True, - raw_content=parsed.raw_content or content, + raw_content=parsed.raw_content or content_text, ) @@ -725,11 +744,14 @@ def _build_verifier_repair_callback(client: object) -> Callable[[str, str, str], repair = build_repair_callback(client) def _repair(schema_json: str, validation_errors: str, raw_response: str) -> str | None: - return repair( + repaired = repair( schema_json, validation_errors, _cap_prompt_text(raw_response, EVAL_SCHEMA_REPAIR_BUDGET_TOKENS), ) + if not repaired: + return None + return _coerce_response_content(repaired) return _repair diff --git a/tests/scripts/test_pr_verifier_structured_output.py b/tests/scripts/test_pr_verifier_structured_output.py index b6013a2df..c708d893d 100644 --- a/tests/scripts/test_pr_verifier_structured_output.py +++ b/tests/scripts/test_pr_verifier_structured_output.py @@ -83,6 +83,89 @@ def test_comparison_runner_repairs_malformed_output() -> None: assert mock_client.invoke.call_count == 2 +def test_comparison_runner_handles_structured_response_content() -> None: + payload = json.dumps(_valid_payload()) + response = _response_with( + [ + {"type": "thinking", "thinking": "reviewing", "signature": "opaque"}, + {"type": "text", "text": payload}, + ] + ) + mock_client = mock.MagicMock() + mock_client.invoke.return_value = response + + runner = pr_verifier.ComparisonRunner( + context="context", + diff=None, + prompt="prompt", + clients=[(mock_client, "anthropic", "claude-sonnet")], + ) + + result = runner.run_single(mock_client, "anthropic", "claude-sonnet") + + assert result.verdict == "PASS" + assert result.raw_content == payload + + +def test_comparison_runner_concatenates_split_text_blocks() -> None: + payload = json.dumps(_valid_payload()) + # Split inside the summary string literal, where an inserted newline would + # be an unescaped control character and invalidate the JSON. + split = payload.index("Looks good.") + 5 + response = _response_with( + [ + {"type": "text", "text": payload[:split]}, + {"type": "text", "text": payload[split:]}, + ] + ) + mock_client = mock.MagicMock() + mock_client.invoke.return_value = response + + runner = pr_verifier.ComparisonRunner( + context="context", + diff=None, + prompt="prompt", + clients=[(mock_client, "anthropic", "claude-sonnet")], + ) + + result = runner.run_single(mock_client, "anthropic", "claude-sonnet") + + assert result.verdict == "PASS" + assert result.raw_content == payload + assert mock_client.invoke.call_count == 1 + + +def test_comparison_runner_normalizes_structured_repair_response() -> None: + payload = json.dumps(_valid_payload()) + malformed = _response_with( + [ + {"type": "thinking", "thinking": "reviewing", "signature": "opaque"}, + {"type": "text", "text": "```json\n" + payload + "\n```"}, + ] + ) + repaired = _response_with( + [ + {"type": "thinking", "thinking": "repairing", "signature": "opaque"}, + {"type": "text", "text": payload}, + ] + ) + mock_client = mock.MagicMock() + mock_client.invoke.side_effect = [malformed, repaired] + + runner = pr_verifier.ComparisonRunner( + context="context", + diff=None, + prompt="prompt", + clients=[(mock_client, "anthropic", "claude-sonnet")], + ) + + result = runner.run_single(mock_client, "anthropic", "claude-sonnet") + + assert result.verdict == "PASS" + assert result.raw_content == payload + assert mock_client.invoke.call_count == 2 + + def test_evaluate_pr_valid_output_no_repair(monkeypatch: pytest.MonkeyPatch) -> None: payload = _valid_payload() good = json.dumps(payload)