Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/autofix-versions.env
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@ MYPY_VERSION=2.3.0
PYTEST_VERSION=9.1.1
PYTEST_COV_VERSION=7.1.0
PYTEST_XDIST_VERSION=3.8.0
COVERAGE_VERSION=7.15.2
COVERAGE_VERSION=7.15.3
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ dev = [
"flake8==7.3.0",
"pytest==9.1.1",
"pytest-cov==7.1.0",
"coverage==7.15.2",
"coverage==7.15.3",
"pytest-xdist==3.8.0",
"packaging>=26.1",

Expand Down
2 changes: 1 addition & 1 deletion requirements.lock
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ colorama==0.4.6 ; sys_platform == 'win32'
# click
# pytest
# tqdm
coverage==7.15.2
coverage==7.15.3
# via
# workflows (pyproject.toml)
# pytest-cov
Expand Down
9 changes: 9 additions & 0 deletions scripts/check_deliberate_break.py
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,15 @@ def verify_spec(
command=list(exc.cmd) if isinstance(exc.cmd, (tuple, list)) else str(exc.cmd),
timeout=exc.timeout,
)
except subprocess.CalledProcessError as exc:
return _json_result(
VERDICT_BROKEN,
reason="tamper-check-failed",
command=list(exc.cmd) if isinstance(exc.cmd, (tuple, list)) else str(exc.cmd),
returncode=exc.returncode,
stdout=exc.stdout,
stderr=exc.stderr,
)

try:
_ensure_pytest_runtime_deps()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,11 +79,17 @@ def build_verdict(output: str) -> dict[str, Any]:
verdict = _normalize_verdict(candidate.get("verdict"))
if not verdict:
continue
candidate_needs_attention = candidate.get("needs_attention")
needs_attention = (
candidate_needs_attention
if isinstance(candidate_needs_attention, bool)
else verdict != "pass"
)
return {
**candidate,
"verdict": verdict,
"source": "structured-json",
"needs_attention": bool(candidate.get("needs_attention", verdict != "pass")),
"needs_attention": needs_attention,
}

return {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@ MYPY_VERSION=2.3.0
PYTEST_VERSION=9.1.1
PYTEST_COV_VERSION=7.1.0
PYTEST_XDIST_VERSION=3.8.0
COVERAGE_VERSION=7.15.2
COVERAGE_VERSION=7.15.3
Comment thread
coderabbitai[bot] marked this conversation as resolved.
9 changes: 9 additions & 0 deletions templates/consumer-repo/scripts/check_deliberate_break.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,15 @@ def verify_spec(
command=list(exc.cmd) if isinstance(exc.cmd, (tuple, list)) else str(exc.cmd),
timeout=exc.timeout,
)
except subprocess.CalledProcessError as exc:
return _json_result(
VERDICT_BROKEN,
reason="tamper-check-failed",
command=list(exc.cmd) if isinstance(exc.cmd, (tuple, list)) else str(exc.cmd),
returncode=exc.returncode,
stdout=exc.stdout,
stderr=exc.stderr,
)

if head_run.returncode != 0:
return _json_result(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@ MYPY_VERSION=2.3.0
PYTEST_VERSION=9.1.1
PYTEST_COV_VERSION=7.1.0
PYTEST_XDIST_VERSION=3.8.0
COVERAGE_VERSION=7.15.2
COVERAGE_VERSION=7.15.3
37 changes: 37 additions & 0 deletions tests/scripts/test_check_deliberate_break.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,43 @@ def test_assertion_tamper_is_flagged(tmp_path, monkeypatch) -> None:
assert result["reason"] == "test-assertion-tamper"


def test_tamper_git_failure_is_broken(tmp_path, monkeypatch) -> None:
repo = tmp_path / "repo"
repo.mkdir()
_init_repo(repo)
_write_app(repo, 0)
_write_test(repo, 0)
base = _commit(repo, "base test")
spec = parse_deliberate_break_spec(
"<!-- deliberate-break: "
"test=tests/test_app.py::test_value "
"test-file=tests/test_app.py "
"break-file=app.py -->"
)
assert spec is not None

def fail_tamper_check(*_args, **_kwargs):
raise subprocess.CalledProcessError(
128,
["git", "diff", f"{base}...HEAD"],
output="",
stderr="bad revision",
)

monkeypatch.setattr(deliberate_break, "_changed_assertions", fail_tamper_check)

result = verify_spec(spec, base=base, cwd=repo)

assert result == {
"verdict": VERDICT_BROKEN,
"reason": "tamper-check-failed",
"command": ["git", "diff", f"{base}...HEAD"],
"returncode": 128,
"stdout": "",
"stderr": "bad revision",
}


def test_new_assertion_in_existing_test_file_is_not_tamper(tmp_path, monkeypatch) -> None:
repo = tmp_path / "repo"
repo.mkdir()
Expand Down
8 changes: 6 additions & 2 deletions tests/tools/test_evaluate_model_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -303,11 +303,15 @@ def test_override_for_unknown_category_is_rejected():
evaluator.evaluate_benchmark(_thin_payload(), policy)


def test_override_must_be_an_object():
@pytest.mark.parametrize(
"invalid_overrides",
[["stale-verifier-claim"], [], ""],
)
def test_override_must_be_an_object(invalid_overrides):
policy = _policy()
policy["profiles"]["verifier-balanced"]["approval_stage"][
"minimum_cases_per_category_overrides"
] = ["stale-verifier-claim"]
] = invalid_overrides

with pytest.raises(ValueError, match="must be an object"):
evaluator.evaluate_benchmark(_thin_payload(), policy)
4 changes: 3 additions & 1 deletion tools/evaluate_model_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,9 @@ def evaluate_benchmark(payload: dict[str, Any], policy: dict[str, Any]) -> dict[
# from realized outcomes (see tools/harvest_verifier_corpus.py) can never be
# grown by the harvester, so holding them to the machine-harvestable floor
# made `approved` unreachable without hand-labelling. See #2819.
raw_overrides = approval.get("minimum_cases_per_category_overrides") or {}
raw_overrides = approval.get("minimum_cases_per_category_overrides")
if raw_overrides is None:
raw_overrides = {}
if not isinstance(raw_overrides, dict):
raise ValueError("approval_stage.minimum_cases_per_category_overrides must be an object")
category_floors: dict[str, int] = {}
Expand Down
Loading