Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion src/coder_eval/criteria/cli_called.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,12 @@ def _check_impl(
facets.append(f"flags={sorted(criterion.flags)}")
wanted = ", ".join(facets)

if score == 1.0:
if score == 1.0 and count == 0 and criterion.max_count == 0:
details = (
f"0 invocation(s) matched ({wanted}); guard unproven: a typo in verb, tool or flags "
"would also match nothing"
)
elif score == 1.0:
details = f"{count} invocation(s) matched ({wanted}); satisfies {bound}"
elif not within_lower:
# A bare count sends the reader to the sandbox; this criterion exists
Expand Down
13 changes: 12 additions & 1 deletion tests/test_cli_called_criterion.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,18 @@ def test_max_count_zero_is_the_negative_guard(self, sandbox_with_log):
min_count=0,
max_count=0,
)
assert SuccessChecker(sandbox).check(forbidden).score == 1.0
result = SuccessChecker(sandbox).check(forbidden)
assert result.score == 1.0
assert "guard unproven" in (result.details or "")
assert "satisfies" not in (result.details or "")

def test_positive_match_keeps_satisfies_wording(self, sandbox_with_log):
sandbox, sandbox_dir = sandbox_with_log
_write_log(sandbox_dir, [_call(["ixp", "fields", "rename", "proj-1"])])
criterion = CliCalledCriterion(description="renamed the field", log=LOG, verb="ixp fields rename")
result = SuccessChecker(sandbox).check(criterion)
assert result.score == 1.0
assert result.details == "1 invocation(s) matched (verb='ixp fields rename'); satisfies min_count=1"

def test_max_count_zero_fails_when_called(self, sandbox_with_log):
sandbox, sandbox_dir = sandbox_with_log
Expand Down