Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions scripts/check_deliberate_break.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import tempfile
from collections.abc import Iterator
from dataclasses import dataclass
from importlib import metadata
from io import BytesIO
from pathlib import Path

Expand All @@ -32,7 +33,8 @@
r"\b(assert|expect\(|pytest\.raises\(|assert\.)\b",
)
DEFAULT_TIMEOUT_SECONDS = 120
PYTEST_RUNTIME_DEPENDENCIES = ("pyyaml==6.0.3",)
PYTEST_RUNTIME_VERSION = "6.0.3"
PYTEST_RUNTIME_DEPENDENCIES = (f"pyyaml=={PYTEST_RUNTIME_VERSION}",)


@dataclass(frozen=True)
Expand Down Expand Up @@ -140,8 +142,10 @@ def _ensure_pytest_runtime_deps() -> None:
Actions ``action_required`` approval wait on workflow-touching PRs.
"""
try:
import yaml # noqa: F401
except ImportError:
installed_version = metadata.version("PyYAML")
except metadata.PackageNotFoundError:
installed_version = None
if installed_version != PYTEST_RUNTIME_VERSION:
subprocess.run(
[
sys.executable,
Expand Down
2 changes: 1 addition & 1 deletion scripts/langchain/pr_verifier.py
Original file line number Diff line number Diff line change
Expand Up @@ -685,7 +685,7 @@ def _text_from_response_content(content: object) -> str | None:
for block in content
if isinstance(block, dict) and isinstance(block.get("text"), str)
]
if text_blocks:
if any(block.strip() for block in text_blocks):
# Concatenate without a separator: a provider may split one JSON
# document across blocks, and an inserted newline inside a string
# literal would make the reassembled payload invalid JSON.
Expand Down
34 changes: 26 additions & 8 deletions tests/scripts/test_check_deliberate_break.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import builtins
import os
import subprocess
import sys
Expand Down Expand Up @@ -260,20 +259,20 @@ def test_cli_skips_without_marker(tmp_path) -> None:
assert "skipped: no deliberate-break marker" in completed.stdout


def test_runtime_dependency_installer_uses_locked_pyyaml(monkeypatch) -> None:
real_import = builtins.__import__
@pytest.mark.parametrize("installed_version", [None, "6.0.2"])
def test_runtime_dependency_installer_uses_locked_pyyaml(monkeypatch, installed_version) -> None:
calls: list[tuple[object, dict[str, object]]] = []

def missing_yaml(name, *args, **kwargs):
if name == "yaml":
raise ImportError("PyYAML missing")
return real_import(name, *args, **kwargs)
def package_version(_name):
if installed_version is None:
raise deliberate_break.metadata.PackageNotFoundError
return installed_version

def record_install(*args, **kwargs):
calls.append((args, kwargs))
return subprocess.CompletedProcess(args[0], 0, "", "")

monkeypatch.setattr(builtins, "__import__", missing_yaml)
monkeypatch.setattr(deliberate_break.metadata, "version", package_version)
monkeypatch.setattr(deliberate_break.subprocess, "run", record_install)

deliberate_break._ensure_pytest_runtime_deps()
Expand All @@ -300,6 +299,25 @@ def record_install(*args, **kwargs):
]


def test_runtime_dependency_installer_accepts_exact_locked_pyyaml(monkeypatch) -> None:
calls: list[object] = []

monkeypatch.setattr(
deliberate_break.metadata,
"version",
lambda _name: deliberate_break.PYTEST_RUNTIME_VERSION,
)
monkeypatch.setattr(
deliberate_break.subprocess,
"run",
lambda *args, **kwargs: calls.append((args, kwargs)),
)

deliberate_break._ensure_pytest_runtime_deps()

assert calls == []


def _sound_spec(repo: Path) -> tuple[str, object]:
_write_app(repo, 0)
base = _commit(repo, "base behavior")
Expand Down
20 changes: 20 additions & 0 deletions tests/scripts/test_pr_verifier_structured_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,26 @@ def test_text_from_response_content_concatenates_split_text_blocks_without_separ
assert pr_verifier._coerce_response_content(blocks) == encoded


def test_text_from_response_content_preserves_blank_blocks_between_text() -> None:
blocks = [
{"type": "text", "text": '{"summary":"not'},
{"type": "text", "text": " "},
{"type": "text", "text": 'safe"}'},
]

assert pr_verifier._text_from_response_content(blocks) == '{"summary":"not safe"}'


def test_text_from_response_content_ignores_blank_text_blocks() -> None:
blocks = [
{"type": "text", "text": " \n"},
{"type": "thinking", "signature": "still thinking"},
]

assert pr_verifier._text_from_response_content(blocks) is None
assert pr_verifier._coerce_response_content(blocks) == json.dumps(blocks, default=str)


def test_evaluate_pr_valid_output_no_repair(monkeypatch: pytest.MonkeyPatch) -> None:
payload = _valid_payload()
good = json.dumps(payload)
Expand Down
11 changes: 11 additions & 0 deletions tests/tools/test_evaluate_model_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,17 @@ def test_override_floor_below_one_is_rejected(floor):
evaluator.evaluate_benchmark(_thin_payload(), policy)


@pytest.mark.parametrize("floor", [1.9, "2", None, True])
def test_override_floor_must_be_an_integer(floor):
policy = _policy()
policy["profiles"]["verifier-balanced"]["approval_stage"][
"minimum_cases_per_category_overrides"
] = {"review-thread-debt": floor}

with pytest.raises(ValueError, match="must be integers"):
evaluator.evaluate_benchmark(_thin_payload(), policy)


def test_override_for_unknown_category_is_rejected():
policy = _policy()
policy["profiles"]["verifier-balanced"]["approval_stage"][
Expand Down
6 changes: 5 additions & 1 deletion tools/evaluate_model_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,11 @@ def evaluate_benchmark(payload: dict[str, Any], policy: dict[str, Any]) -> dict[
category_floors: dict[str, int] = {}
for category in required_categories:
floor = raw_overrides.get(category, minimum_per_category)
floor = int(floor)
if type(floor) is not int:
raise ValueError(
"minimum_cases_per_category_overrides values must be integers "
f"(got {floor!r} for {category!r})"
)
if floor < 1:
raise ValueError(
"minimum_cases_per_category_overrides values must be >= 1 "
Expand Down
Loading