From 4a74e932b00e5aee0d05ae18107d16b59688cf20 Mon Sep 17 00:00:00 2001 From: Mohit Upadhyay Date: Fri, 7 Aug 2026 15:01:11 +0000 Subject: [PATCH] Report when the gold check had nothing to check against `Task.validate` returns `True` unconditionally, so a taskset that does not override it passes the gold check for every item without anything having been checked. Of the five in-tree tasksets defining a `Task` subclass, only `lean` overrides `validate`; `harbor`, `openenv`, `textarena` and `nemo_gym` inherit the base implementation. For those, `validate --only-gold` reports `valid_rate: 1.0`, which reads as a passing check but measures nothing. Mark such rows with `checked: false`, count them as `gold_unchecked` in the summary, and warn once per taskset naming the classes involved. Validation behaviour is unchanged; this only distinguishes "checked and valid" from "never checked". --- verifiers/v1/cli/validate.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/cli/validate.py b/verifiers/v1/cli/validate.py index c1fae6e1e3..90f403ac57 100644 --- a/verifiers/v1/cli/validate.py +++ b/verifiers/v1/cli/validate.py @@ -154,6 +154,22 @@ def load_results( return rows, owed +def gold_implemented(task: Task) -> bool: + """Whether the taskset actually wrote a gold check. + + `Task.validate` returns `True` unconditionally, so a task that does not override it + passes the gold check for every item without anything having been checked.""" + return type(task).validate is not Task.validate + + +def _gold_row(row: ResultRow) -> ResultRow | None: + """The gold row inside a result row, whichever mode produced it.""" + if row.get("mode") == "gold": + return row + gold = row.get("gold") + return gold if isinstance(gold, dict) else None + + def summarize(rows: Sequence[ResultRow], total: int, mode: str) -> dict[str, Any]: counts = Counter(row.get("reason") for row in rows) missing = max(0, total - len(rows)) @@ -169,6 +185,12 @@ def summarize(rows: Sequence[ResultRow], total: int, mode: str) -> dict[str, Any "outcomes": outcomes, "valid_rate": round(outcomes["valid"] / total, 6) if total else None, } + if mode in ("gold", "all"): + golds = [gold for gold in map(_gold_row, rows) if gold is not None] + # Gold rows that were reported valid without a gold check to be valid against. + summary["gold_unchecked"] = sum( + 1 for gold in golds if gold.get("checked") is False + ) if mode == "all": checks: dict[str, dict[str, int]] = {} for check in ("gold", "setup"): @@ -271,7 +293,9 @@ async def _run_gold(task: Task, config: ValidateConfig) -> ResultRow: logger.warning( "runtime teardown failed (task %s)", task.data.idx, exc_info=True ) - return _row(task, "gold", valid, exc, start) + # A task that never overrode `validate` is valid against nothing; say so on the row + # rather than letting the base class's unconditional `True` read as a passing check. + return {**_row(task, "gold", valid, exc, start), "checked": gold_implemented(task)} async def _run_setup(task: Task, config: ValidateConfig) -> ResultRow: @@ -406,6 +430,15 @@ async def run_validate(config: ValidateConfig) -> list[dict]: checks, ) logger.info("results: %s", out) + if mode in ("gold", "all"): + unchecked = sorted({type(t).__name__ for t in tasks if not gold_implemented(t)}) + if unchecked: + logger.warning( + "no gold check: %s does not override Task.validate, which returns True " + "unconditionally - the gold check will pass every task without checking " + "anything (see gold_unchecked in the summary)", + ", ".join(unchecked), + ) sem = asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None states = [TaskProgress(idx=t.data.idx, name=t.data.name) for t in tasks]