From b6cf7710b54266349f33fb74d4b0184640942288 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Mon, 3 Aug 2026 21:23:18 +0200 Subject: [PATCH 1/3] refactor(v1): reuse pydantic type adapters --- tests/v1/test_judges.py | 20 ++- verifiers/v1/cli/output.py | 8 +- verifiers/v1/configs/judge.py | 4 +- verifiers/v1/envs/agentic_judge/env.py | 95 ++--------- verifiers/v1/harnesses/bash/program.py | 39 +++-- verifiers/v1/harnesses/browser_use/program.py | 31 ++-- verifiers/v1/harnesses/null/program.py | 39 +++-- verifiers/v1/harnesses/rlm/harness.py | 49 +++--- verifiers/v1/interception/server.py | 6 +- verifiers/v1/judges/rubric.py | 149 ++++++++++-------- verifiers/v1/legacy.py | 26 +-- verifiers/v1/session.py | 8 + 12 files changed, 236 insertions(+), 238 deletions(-) diff --git a/tests/v1/test_judges.py b/tests/v1/test_judges.py index 49369e4f0b..ab803d2e6e 100644 --- a/tests/v1/test_judges.py +++ b/tests/v1/test_judges.py @@ -9,6 +9,7 @@ from pydantic import Field import verifiers.v1 as vf +from verifiers.v1.envs.agentic_judge import ScoreConfig from verifiers.v1.graph import MessageNode from verifiers.v1.judge import Judge, JudgeResponse from verifiers.v1.types import AssistantMessage, UserMessage @@ -511,8 +512,23 @@ def test_rubric_rejects_bad_files(tmp_path): ).criteria # negative/NaN/inf weights would invert a criterion or corrupt the weighted mean for weight in (-1.0, float("nan"), float("inf")): - with pytest.raises(ValueError, match="negative or non-finite"): + with pytest.raises( + ValueError, match="greater than or equal to 0|finite number" + ): _ = rubric_judge(tmp_path, weights={"mentions_paris": weight}).criteria + with pytest.raises(ValueError, match="non-finite total"): + _ = rubric_judge( + tmp_path, weights={"mentions_paris": 1e308, "is_polite": 1e308} + ).criteria + + +@pytest.mark.parametrize("weight", [float("nan"), float("inf"), float("-inf")]) +def test_judge_composition_weights_are_finite(weight): + with pytest.raises(ValueError, match="finite number"): + vf.JudgeConfig(weight=weight) + for field in ("task_weight", "judge_weight"): + with pytest.raises(ValueError, match="finite number"): + ScoreConfig.model_validate({field: weight}) async def test_rubric_score(tmp_path, fake_judge_model): @@ -569,7 +585,7 @@ async def off_menu(self, messages, *, trace=None, schema=None, parse=None, **s): def test_rubric_choices_validation(tmp_path): - with pytest.raises(ValueError, match="at least two"): + with pytest.raises(ValueError, match="at least 2"): _ = rubric_judge( tmp_path, body='[[criteria]]\nname = "x"\ntext = "t"\nchoices = ["only"]\n' ).criteria diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py index ec214568b5..e6da861811 100644 --- a/verifiers/v1/cli/output.py +++ b/verifiers/v1/cli/output.py @@ -12,6 +12,7 @@ import asyncio import json +from functools import cache from pathlib import Path import tomli_w @@ -29,6 +30,9 @@ CONFIG_FILE = "config.toml" """Filename a run's resolved config is written to (re-runnable via `@ config.toml`).""" +# Compiling an adapter is the expensive part; run output reuses only a few model classes. +_type_adapter = cache(TypeAdapter) + def output_path(config: EvalConfig) -> Path: """Where this run writes: `outputs/----/` (or the explicit @@ -72,7 +76,7 @@ def save_config(config: BaseModel, results_dir: Path) -> None: def write_episode(results_dir: Path, episode: Episode) -> None: """Serialize and append one rollout episode in the worker thread.""" # Preserve fields declared by typed Trace subclasses nested in the episode. - data = TypeAdapter(type(episode)).dump_json(episode, exclude_none=True) + data = _type_adapter(type(episode)).dump_json(episode, exclude_none=True) with (results_dir / TRACES_FILE).open("ab") as f: f.write(data + b"\n") @@ -88,7 +92,7 @@ def read_episodes(results_dir: Path, trace_type: type) -> list[Episode]: `trace_type` (`Trace[WireTaskData, ...]` reads any taskset's file without importing it). A pre-episode line (one bare trace) is wrapped as a single-trace record, so both file generations read uniformly.""" - trace_adapter = TypeAdapter(trace_type) + trace_adapter = _type_adapter(trace_type) episodes: list[Episode] = [] with (results_dir / TRACES_FILE).open(encoding="utf-8") as f: for line in f: diff --git a/verifiers/v1/configs/judge.py b/verifiers/v1/configs/judge.py index b698d1a32c..92158353e7 100644 --- a/verifiers/v1/configs/judge.py +++ b/verifiers/v1/configs/judge.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Any -from pydantic import BaseModel, SerializeAsAny +from pydantic import BaseModel, FiniteFloat, SerializeAsAny from verifiers.v1.clients import BaseClientConfig from verifiers.v1.types import ID, SamplingConfig @@ -17,7 +17,7 @@ class JudgeConfig(BaseClientConfig): """Plugin id; empty for a judge called directly by task code.""" name: str = "" """Reward key override for a plugged judge.""" - weight: float = 1.0 + weight: FiniteFloat = 1.0 model: str = "openai/gpt-5.4-nano" sampling: SamplingConfig = SamplingConfig() prompt: Path | None = None diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index a0b2b5ddce..a71fabb50d 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -14,14 +14,18 @@ """ import json -import math import re -import tomllib from pathlib import Path -from pydantic import BaseModel, Field, field_validator +from pydantic import FiniteFloat import verifiers.v1 as vf +from verifiers.v1.judges.rubric import ( + Criterion, + RubricVerdicts, + _load_criteria, + _score_verdicts, +) from verifiers.v1.utils.compile import validate_pairing VERDICT_FILE = "/tmp/verdict.json" @@ -38,29 +42,6 @@ {prompt}""" -class Criterion(BaseModel): - """One rubric criterion — the plugged rubric judge's format, mirrored so the - same `criteria` files grade both judges.""" - - name: str - """Key for the criterion's metric (`judge/`).""" - text: str - weight: float = 1.0 - """The criterion's share of the reward.""" - choices: list[str] = Field(default_factory=lambda: ["no", "yes"]) - """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest - evenly spaced by rank. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates.""" - - @field_validator("choices") - @classmethod - def _check_choices(cls, v: list[str]) -> list[str]: - if len(v) < 2: - raise ValueError(f"`choices` needs at least two options, got {v}") - if len(set(v)) != len(v): - raise ValueError(f"`choices` has duplicate options: {v}") - return v - - SOLVED = Criterion( name="solved", text="The task is fully solved: what the task asked for is achieved, and " @@ -210,7 +191,7 @@ async def finalize(self, trace: vf.Trace, runtime: vf.Runtime) -> None: f"the judge wrote no verdict to {VERDICT_FILE}; its final act must " 'be writing {"verdicts": [{"name", "reason", "verdict"}, ...]} there' ) from e - trace.info["verdict"] = json.loads(raw) + trace.info["verdict"] = RubricVerdicts.model_validate_json(raw).model_dump() class TextFile(vf.BaseConfig): @@ -259,38 +240,16 @@ def build_hint(self) -> str | None: def criteria(self) -> list[Criterion]: if self.rubric is None: return [SOLVED] - text = self.rubric.read_text(encoding="utf-8") - data = ( - tomllib.loads(text) - if self.rubric.suffix.lower() == ".toml" - else json.loads(text) - ) - items = data.get("criteria", []) if isinstance(data, dict) else data - criteria = [Criterion.model_validate(item) for item in items] - if not criteria: - raise ValueError(f"rubric file '{self.rubric}' lists no criteria") - names = [criterion.name for criterion in criteria] - if len(set(names)) != len(names): - raise ValueError( - f"rubric file '{self.rubric}' has duplicate criterion names" - ) - if bad := [c.name for c in criteria if not 0 <= c.weight < math.inf]: - raise ValueError( - f"rubric '{self.rubric}' has negative or non-finite criterion " - f"weights: {bad}" - ) - if sum(criterion.weight for criterion in criteria) <= 0: - raise ValueError(f"rubric '{self.rubric}' has no positive criterion weight") - return criteria + return _load_criteria(self.rubric) class ScoreConfig(vf.BaseConfig): """How the judge's verdict composes with the taskset's own rewards on the solver's trace. Judge-only by default.""" - task_weight: float = 0.0 + task_weight: FiniteFloat = 0.0 """Scale applied to the taskset's own rewards; 1 keeps them next to the verdict.""" - judge_weight: float = 1.0 + judge_weight: FiniteFloat = 1.0 """Weight of the judge's verdict in the solver's reward.""" @@ -363,37 +322,9 @@ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: if "judge" not in by_agent: return solution, verdict = by_agent["solver"], by_agent["judge"] - data = verdict.info.get("verdict") - if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list): - raise TypeError( - f"no verdicts on the judge's trace (expected {VERDICT_FILE} with a " - '"verdicts" list)' - ) + verdicts = RubricVerdicts.model_validate(verdict.info.get("verdict")).verdicts criteria = self.config.task.criteria() - by_criterion = {c.name: c for c in criteria} - answers: dict[str, str] = {} - for entry in data["verdicts"]: - if not isinstance(entry, dict): - raise TypeError(f"verdict entry {entry!r} is not an object") - name = str(entry.get("name")) - if name in answers: - # Contradictory duplicates must not collapse to whichever came last. - raise ValueError(f"judge answered criterion {name!r} more than once") - answers[name] = str(entry.get("verdict")) - if sorted(answers) != sorted(by_criterion): - raise ValueError( - f"judge verdicts name {sorted(answers)}, expected the rubric's " - f"{sorted(by_criterion)}" - ) - scores: dict[str, float] = {} - for name, answer in answers.items(): - choices = by_criterion[name].choices - # An off-menu answer is a judge failure, not a zero score. - if answer not in choices: - raise ValueError( - f"judge answered {answer!r} for '{name}', expected one of {choices}" - ) - scores[name] = choices.index(answer) / (len(choices) - 1) + scores = _score_verdicts(verdicts, criteria, "the rubric's") for criterion in criteria: solution.record_metric(f"judge/{criterion.name}", scores[criterion.name]) if self.config.score.task_weight != 1.0: diff --git a/verifiers/v1/harnesses/bash/program.py b/verifiers/v1/harnesses/bash/program.py index 643963854f..35020a512d 100644 --- a/verifiers/v1/harnesses/bash/program.py +++ b/verifiers/v1/harnesses/bash/program.py @@ -1,18 +1,25 @@ # /// script # requires-python = ">=3.10" -# dependencies = ["openai", "mcp>=1.24.0,<2", "httpx", "tenacity"] +# dependencies = [ +# "openai", +# "mcp>=1.24.0,<2", +# "httpx", +# "tenacity", +# "pydantic>=2.13.4", +# ] # /// """Secrets arrive through argv so local tool subprocesses do not inherit them.""" import argparse import asyncio -import json import subprocess from contextlib import AsyncExitStack, asynccontextmanager, suppress from pathlib import Path +from typing import Any import httpx from openai import AsyncOpenAI +from pydantic import TypeAdapter, ValidationError from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter SERPER_URL = "https://google.serper.dev/search" @@ -20,6 +27,9 @@ MCP_CALL_ATTEMPTS = 6 MCP_TIMEOUT = 600.0 +JSON_OBJECT_ADAPTER = TypeAdapter(dict[str, Any]) +MESSAGES_ADAPTER = TypeAdapter(list[dict[str, Any]]) + BASH_TOOL = { "type": "function", @@ -321,9 +331,9 @@ async def main() -> None: path = Path(args.initial_messages_file) payload = path.read_bytes() path.unlink() - initial = json.loads(payload) + initial = MESSAGES_ADAPTER.validate_json(payload) client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key) - config = json.loads(args.mcp_config or "{}") + config = JSON_OBJECT_ADAPTER.validate_json(args.mcp_config or "{}") tools = [BASH_TOOL] reserved = {"bash"} if args.edit: @@ -355,24 +365,19 @@ async def main() -> None: for call in message.tool_calls: name = call.function.name try: - tool_args = json.loads(call.function.arguments or "{}") - except json.JSONDecodeError as e: - messages.append( - { - "role": "tool", - "tool_call_id": call.id, - "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON", - } + tool_args = JSON_OBJECT_ADAPTER.validate_json( + call.function.arguments or "{}" ) - continue - # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls - # below assume a dict, so reject anything else as a tool error rather than crashing. - if not isinstance(tool_args, dict): + except ValidationError as e: messages.append( { "role": "tool", "tool_call_id": call.id, - "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object", + "content": ( + "error: invalid tool arguments " + f"({e.errors(include_url=False)[0]['msg']}); " + "resend as a JSON object" + ), } ) continue diff --git a/verifiers/v1/harnesses/browser_use/program.py b/verifiers/v1/harnesses/browser_use/program.py index 1f790c9ded..1539046e84 100644 --- a/verifiers/v1/harnesses/browser_use/program.py +++ b/verifiers/v1/harnesses/browser_use/program.py @@ -6,6 +6,7 @@ # "mcp>=1.24.0,<2", # "httpx", # "tenacity", +# "pydantic>=2.13.4", # ] # /// """A chat loop whose one local tool drives a real Chromium over CDP. @@ -18,7 +19,6 @@ import argparse import asyncio -import json import os import re import shutil @@ -33,11 +33,15 @@ import httpx from openai import AsyncOpenAI +from pydantic import TypeAdapter, ValidationError from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter MCP_CALL_ATTEMPTS = 6 MCP_TIMEOUT = 600.0 +JSON_OBJECT_ADAPTER = TypeAdapter(dict[str, Any]) +MESSAGES_ADAPTER = TypeAdapter(list[dict[str, Any]]) + BROWSER_TOOL_TIMEOUT = 3600 """Matches the bash harness's command timeout.""" @@ -322,7 +326,7 @@ async def main() -> None: path = Path(args.initial_messages_file) payload = path.read_bytes() path.unlink() - initial = json.loads(payload) + initial = MESSAGES_ADAPTER.validate_json(payload) state_dir = Path(args.state_dir) state_dir.mkdir(parents=True, exist_ok=True) endpoint = ( @@ -337,7 +341,7 @@ async def main() -> None: ): endpoint, } client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key) - config = json.loads(args.mcp_config or "{}") + config = JSON_OBJECT_ADAPTER.validate_json(args.mcp_config or "{}") tools = [BROWSER_TOOL] reserved = {"browser"} mcp_tools, dispatch, servers = ( @@ -363,24 +367,19 @@ async def main() -> None: for call in message.tool_calls: name = call.function.name try: - tool_args = json.loads(call.function.arguments or "{}") - except json.JSONDecodeError as e: - messages.append( - { - "role": "tool", - "tool_call_id": call.id, - "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON", - } + tool_args = JSON_OBJECT_ADAPTER.validate_json( + call.function.arguments or "{}" ) - continue - # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls - # below assume a dict, so reject anything else as a tool error rather than crashing. - if not isinstance(tool_args, dict): + except ValidationError as e: messages.append( { "role": "tool", "tool_call_id": call.id, - "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object", + "content": ( + "error: invalid tool arguments " + f"({e.errors(include_url=False)[0]['msg']}); " + "resend as a JSON object" + ), } ) continue diff --git a/verifiers/v1/harnesses/null/program.py b/verifiers/v1/harnesses/null/program.py index d5231de072..efc9fc866d 100644 --- a/verifiers/v1/harnesses/null/program.py +++ b/verifiers/v1/harnesses/null/program.py @@ -1,22 +1,32 @@ # /// script # requires-python = ">=3.11" -# dependencies = ["openai", "mcp>=1.24.0,<2", "httpx", "tenacity"] +# dependencies = [ +# "openai", +# "mcp>=1.24.0,<2", +# "httpx", +# "tenacity", +# "pydantic>=2.13.4", +# ] # /// """The interception endpoint and secret arrive through argv rather than the environment.""" import argparse import asyncio -import json from contextlib import AsyncExitStack, asynccontextmanager, suppress from pathlib import Path +from typing import Any import httpx from openai import AsyncOpenAI +from pydantic import TypeAdapter, ValidationError from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter MCP_CALL_ATTEMPTS = 6 MCP_TIMEOUT = 600.0 +JSON_OBJECT_ADAPTER = TypeAdapter(dict[str, Any]) +MESSAGES_ADAPTER = TypeAdapter(list[dict[str, Any]]) + async def chat( client: AsyncOpenAI, model: str, messages: list[dict], tools: list[dict] @@ -158,13 +168,13 @@ async def main() -> None: path = Path(args.initial_messages_file) payload = path.read_bytes() path.unlink() - initial = json.loads(payload) + initial = MESSAGES_ADAPTER.validate_json(payload) client = AsyncOpenAI( base_url=args.base_url, api_key=args.api_key, timeout=httpx.Timeout(None, connect=5.0), ) - config = json.loads(args.mcp_config or "{}") + config = JSON_OBJECT_ADAPTER.validate_json(args.mcp_config or "{}") if config.get("mcpServers"): # Bound only tool enumeration; each session is opened and closed within this task. async with asyncio.timeout(60): @@ -188,24 +198,19 @@ async def main() -> None: for call in message.tool_calls: name = call.function.name try: - tool_args = json.loads(call.function.arguments or "{}") - except json.JSONDecodeError as e: - messages.append( - { - "role": "tool", - "tool_call_id": call.id, - "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON", - } + tool_args = JSON_OBJECT_ADAPTER.validate_json( + call.function.arguments or "{}" ) - continue - # Valid JSON can still be a non-object (`[]`, `42`, `null`); the MCP dispatch - # assumes a dict, so reject anything else as a tool error rather than crashing. - if not isinstance(tool_args, dict): + except ValidationError as e: messages.append( { "role": "tool", "tool_call_id": call.id, - "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object", + "content": ( + "error: invalid tool arguments " + f"({e.errors(include_url=False)[0]['msg']}); " + "resend as a JSON object" + ), } ) continue diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index e8e84b55ed..b6a3bacd0d 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -4,9 +4,19 @@ import logging import random import shlex -from typing import Literal - -from pydantic import Field, model_validator +from typing import Annotated, Literal, NotRequired + +from pydantic import ( + Field, + FiniteFloat, + OnErrorOmit, + PositiveInt, + Strict, + TypeAdapter, + ValidationError, + model_validator, +) +from typing_extensions import TypedDict from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import HarnessConfig @@ -21,6 +31,13 @@ BuiltinSkill = Literal["edit", "search"] + +class _SessionMeta(TypedDict): + metrics: NotRequired[dict[str, OnErrorOmit[Annotated[FiniteFloat, Strict()]]]] + + +_SESSION_META_ADAPTER = TypeAdapter(_SessionMeta) + RLM_REPO = "github.com/PrimeIntellect-ai/rlm.git" # rlm writes its session under $RLM_HOME/sessions//; point it at a workdir- # relative dir so it stays in the runtime (and is cleaned up with the workdir). @@ -38,26 +55,18 @@ class RLMHarnessConfig(HarnessConfig): builtin_skills: list[BuiltinSkill] = Field(default_factory=list) """Built-in rlm skills to enable (RLM_SKILLS), e.g. `["edit"]`; empty enables none. The tool set is fixed (ipython); the base `skills` field takes SKILL.md paths.""" - summarize_at_tokens: int | tuple[int, int] | None = None + summarize_at_tokens: PositiveInt | tuple[PositiveInt, PositiveInt] | None = None """Auto-compaction threshold (RLM_SUMMARIZE_AT_TOKENS): compact the context once it grows past this many tokens. An int is a fixed threshold; a `(lo, hi)` pair draws a per-group threshold (seeded by the task index, so a task's rollouts share one draw and tasks vary). `None` disables auto-compaction; ints must be positive.""" @model_validator(mode="after") - def validate_limits(self) -> "RLMHarnessConfig": + def validate_range(self) -> "RLMHarnessConfig": value = self.summarize_at_tokens - if isinstance(value, tuple): - lo, hi = value - if lo <= 0 or hi <= 0: - raise ValueError("`summarize_at_tokens` range bounds must be positive.") - if lo > hi: - raise ValueError( - "`summarize_at_tokens` range must be (lo, hi) with lo <= hi." - ) - elif value is not None and value <= 0: + if isinstance(value, tuple) and value[0] > value[1]: raise ValueError( - "`summarize_at_tokens` must be positive, or None to disable." + "`summarize_at_tokens` range must be (lo, hi) with lo <= hi." ) return self @@ -157,11 +166,7 @@ async def rlm(self, runtime: Runtime) -> dict[str, float]: if result.exit_code != 0 or not result.stdout.strip(): return {} try: - meta = json.loads(result.stdout) - except json.JSONDecodeError: + meta = _SESSION_META_ADAPTER.validate_json(result.stdout) + except ValidationError: return {} - return { - key: float(value) - for key, value in meta.get("metrics", {}).items() - if isinstance(value, (int, float)) and not isinstance(value, bool) - } + return meta.get("metrics", {}) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index b3582d680b..9898a5fba4 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -30,7 +30,7 @@ from typing import Literal from aiohttp import web -from pydantic import TypeAdapter, ValidationError +from pydantic import ValidationError from pydantic_core import PydanticSerializationError, from_json, to_json from verifiers.v1 import graph @@ -737,7 +737,7 @@ async def handle_state_get(self, request: web.Request) -> web.Response: state = session.trace.state return web.Response( # TypeAdapter emits UTF-8 bytes directly, avoiding a JSON str copy in aiohttp. - body=TypeAdapter(type(state)).dump_json(state), + body=session.state_adapter.dump_json(state), content_type="application/json", charset="utf-8", ) @@ -768,7 +768,7 @@ async def handle_state_put(self, request: web.Request) -> web.Response: state_cls = type(session.trace.state) raw = await request.read() try: - new_state = state_cls.model_validate_json(raw) + new_state = session.state_adapter.validate_json(raw) except ValidationError as e: # Reject malformed, over-nested, or mismatched state before it enters the shared channel. logger.warning("state PUT rejected: id=%s %s", session.trace.id, e) diff --git a/verifiers/v1/judges/rubric.py b/verifiers/v1/judges/rubric.py index 35dadae665..f41de6dd06 100644 --- a/verifiers/v1/judges/rubric.py +++ b/verifiers/v1/judges/rubric.py @@ -2,14 +2,13 @@ import asyncio import json -import math import re import tomllib from functools import cached_property from pathlib import Path -from typing import cast +from typing import Annotated, cast -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, BeforeValidator, Field, TypeAdapter, field_validator from verifiers.v1.configs.judge import JudgeConfig from verifiers.v1.judge import Judge, JudgeView, judge_question, judge_response @@ -17,6 +16,8 @@ from verifiers.v1.trace import Trace from verifiers.v1.types import ID +_CriterionWeight = Annotated[float, Field(ge=0, allow_inf_nan=False)] + RUBRIC_PROMPT = (Path(__file__).resolve().parent / "rubric.txt").read_text( encoding="utf-8" ) @@ -60,29 +61,73 @@ class Criterion(BaseModel): name: str """Key for the criterion's metric (`/`) and its `weights` override.""" text: str - weight: float = 1.0 + weight: _CriterionWeight = 1.0 """The criterion's share of the reward (overridable per name via `weights` in config).""" - choices: list[str] = Field(default_factory=lambda: ["no", "yes"]) + choices: list[str] = Field(default_factory=lambda: ["no", "yes"], min_length=2) """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest evenly spaced by rank. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates.""" @field_validator("choices") @classmethod def _check_choices(cls, v: list[str]) -> list[str]: - if len(v) < 2: - raise ValueError(f"`choices` needs at least two options, got {v}") if len(set(v)) != len(v): raise ValueError(f"`choices` has duplicate options: {v}") return v +_CRITERIA_ADAPTER = TypeAdapter( + Annotated[ + list[Criterion], + BeforeValidator( + lambda value: ( + value.get("criteria", []) if isinstance(value, dict) else value + ) + ), + ] +) + + +def _load_criteria( + path: Path, weights: dict[str, _CriterionWeight] | None = None +) -> list[Criterion]: + """Load either supported rubric shape and apply validated config overrides.""" + text = path.read_text(encoding="utf-8") + criteria = ( + _CRITERIA_ADAPTER.validate_python(tomllib.loads(text)) + if path.suffix.lower() == ".toml" + else _CRITERIA_ADAPTER.validate_json(text) + ) + if not criteria: + raise ValueError(f"rubric file '{path}' lists no criteria") + names = [criterion.name for criterion in criteria] + if len(set(names)) != len(names): + raise ValueError(f"rubric file '{path}' has duplicate criterion names") + overrides = weights or {} + if unknown := set(overrides) - set(names): + raise ValueError( + f"`weights` overrides name no criterion in '{path}': {sorted(unknown)}" + ) + criteria = [ + criterion.model_copy( + update={"weight": overrides.get(criterion.name, criterion.weight)} + ) + for criterion in criteria + ] + total = sum(criterion.weight for criterion in criteria) + if total == float("inf"): + raise ValueError(f"rubric '{path}' has a non-finite total criterion weight") + if total <= 0: + raise ValueError(f"rubric '{path}' has no positive criterion weight") + return criteria + + class RubricJudgeConfig(JudgeConfig): id: ID = "rubric" """Pinned to the built-in, so a code-level default entry needs no explicit id.""" path: Path """A `.toml` or `.json` file containing a `criteria` list. Relative paths resolve against the evaluation's working directory.""" - weights: dict[str, float] = Field(default_factory=dict) + weights: dict[str, _CriterionWeight] = Field(default_factory=dict) """Per-criterion weight overrides by criterion name (config wins over the file).""" question_field: str = "" """Task field to fill the prompt's `{question}`; empty = the task's prompt rendered as @@ -96,7 +141,7 @@ class RubricJudgeConfig(JudgeConfig): """How much of the rollout fills `{response}` (see `JudgeView`). Defaults to the whole transcript — rubric criteria typically grade the process (tool use, citations, intermediate steps), not just the final answer.""" - max_criteria: int | None = None + max_criteria: int | None = Field(default=None, ge=1) """How many criteria to grade per judge call. `None` (default) grades all criteria in one call. `1` sends one call per criterion (n independent judges); `k` batches them k-at-a-time. Batches are graded concurrently and merged. Smaller batches trade more calls for focus/ @@ -119,45 +164,43 @@ class RubricVerdicts(BaseModel): verdicts: list[CriterionVerdict] +def _score_verdicts( + verdicts: list[CriterionVerdict], + criteria: list[Criterion], + expected: str, +) -> dict[str, float]: + """Validate a complete named verdict set and normalize its ordered choices.""" + by_criterion = {criterion.name: criterion for criterion in criteria} + answers: dict[str, str] = {} + for verdict in verdicts: + if verdict.name in answers: + raise ValueError( + f"judge answered criterion {verdict.name!r} more than once" + ) + answers[verdict.name] = verdict.verdict + if sorted(answers) != sorted(by_criterion): + raise ValueError( + f"judge verdicts name {sorted(answers)}, expected {expected} " + f"{sorted(by_criterion)}" + ) + scores: dict[str, float] = {} + for name, answer in answers.items(): + choices = by_criterion[name].choices + if answer not in choices: + raise ValueError( + f"judge answered {answer!r} for '{name}', expected one of {choices}" + ) + scores[name] = normalize_choice(answer, choices) + return scores + + class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]): prompt = RUBRIC_PROMPT schema = RubricVerdicts @cached_property def criteria(self) -> list[Criterion]: - path = self.config.path - text = path.read_text(encoding="utf-8") - data = ( - tomllib.loads(text) if path.suffix.lower() == ".toml" else json.loads(text) - ) - items = data.get("criteria", []) if isinstance(data, dict) else data - criteria = [Criterion.model_validate(item) for item in items] - if not criteria: - raise ValueError(f"rubric file '{path}' lists no criteria") - names = [criterion.name for criterion in criteria] - if len(set(names)) != len(names): - raise ValueError(f"rubric file '{path}' has duplicate criterion names") - if unknown := set(self.config.weights) - set(names): - raise ValueError( - f"`weights` overrides name no criterion in '{path}': {sorted(unknown)}" - ) - criteria = [ - criterion.model_copy( - update={ - "weight": self.config.weights.get(criterion.name, criterion.weight) - } - ) - for criterion in criteria - ] - if bad := [c.name for c in criteria if not 0 <= c.weight < math.inf]: - # A negative weight would invert a criterion (pushing the reward out of [0, 1]); - # NaN/inf (which json.loads accepts) would corrupt the weighted mean. - raise ValueError( - f"rubric '{path}' has negative or non-finite criterion weights: {bad}" - ) - if sum(criterion.weight for criterion in criteria) <= 0: - raise ValueError(f"rubric '{path}' has no positive criterion weight") - return criteria + return _load_criteria(self.config.path, self.config.weights) async def grade_batch( self, task: TaskData, trace: Trace, batch: list[Criterion] @@ -205,30 +248,12 @@ def render(c: Criterion) -> str: f"judge returned no verdicts JSON object: {result.text!r}" ) verdicts = RubricVerdicts.model_validate(obj).verdicts - # Exactly one verdict per criterion in the batch, matched by name — anything else is a - # judge failure and must error the rollout, not score the model (see `judge_verdict`). - by_criterion = {c.name: c for c in batch} - if sorted(v.name for v in verdicts) != sorted(by_criterion): - raise ValueError( - f"judge verdicts name {sorted(v.name for v in verdicts)}, expected the " - f"batch's {sorted(by_criterion)}" - ) - scores: dict[str, float] = {} - for v in verdicts: - choices = by_criterion[v.name].choices - # An off-menu answer is a judge failure, not a zero score. - if v.verdict not in choices: - raise ValueError( - f"judge answered {v.verdict!r} for '{v.name}', expected one of {choices}" - ) - scores[v.name] = normalize_choice(v.verdict, choices) - return scores + # A malformed verdict is a judge failure and must error the rollout, not score the model. + return _score_verdicts(verdicts, batch, "the batch's") async def score(self, task: TaskData, trace: Trace) -> float: criteria = self.criteria k = self.config.max_criteria - if k is not None and k < 1: - raise ValueError(f"`max_criteria` must be >= 1 or None, got {k}") batches = ( [criteria] if k is None diff --git a/verifiers/v1/legacy.py b/verifiers/v1/legacy.py index a7870aab0b..abaea20bad 100644 --- a/verifiers/v1/legacy.py +++ b/verifiers/v1/legacy.py @@ -16,11 +16,11 @@ import contextlib import logging from pathlib import Path -from typing import Any +from typing import Annotated, Any import zmq import zmq.asyncio -from pydantic import ValidationError +from pydantic import BeforeValidator, OnErrorOmit, TypeAdapter, ValidationError from verifiers.v1 import graph from verifiers.v1.configs.agent import AgentConfig @@ -76,19 +76,19 @@ def _as_dict(obj: Any) -> Any: return obj +_TOOLS_ADAPTER = TypeAdapter( + list[OnErrorOmit[Annotated[Tool, BeforeValidator(_as_dict)]]] +) + + def _to_v1_tools(raw: Any) -> list[Tool] | None: """Map v0 ``RolloutOutput.tool_defs`` onto ``Trace.tools``. The v0 and v1 ``Tool`` shapes are identical (name/description/parameters/strict), so this is a re-validation; malformed entries are dropped rather than failing the whole trace mapping.""" - defs: list[Tool] = [] - for t in raw or []: - t = _as_dict(t) - if not isinstance(t, dict): - continue - try: - defs.append(Tool.model_validate(t)) - except ValidationError: - continue + try: + defs = _TOOLS_ADAPTER.validate_python(raw or []) + except ValidationError: + return None return defs or None @@ -276,8 +276,8 @@ def rollout_output_to_trace(out: dict, task_idx: int) -> Trace: # base task type above. agent=AgentInfo(config=AgentConfig()), tools=_to_v1_tools(out.get("tool_defs")) or [], - rewards={"reward": Reward(score=float(out.get("reward") or 0.0))}, - metrics={k: float(v) for k, v in (out.get("metrics") or {}).items()}, + rewards={"reward": Reward(score=out.get("reward") or 0.0)}, + metrics=out.get("metrics") or {}, info=dict(out.get("info") or {}), is_completed=bool(out.get("is_completed", True)), # Bridged rollouts are complete by construction; the sentinel mirrors diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index 3aa65f56ba..8e8611208e 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -11,8 +11,11 @@ import logging from collections.abc import Awaitable, Callable from dataclasses import dataclass, field +from functools import cached_property from typing import TYPE_CHECKING +from pydantic import TypeAdapter + from verifiers.v1.clients import Client, ModelContext from verifiers.v1.trace import Trace @@ -96,6 +99,11 @@ class RolloutSession: its client disconnects, so a request whose program died at teardown would keep driving the exchange (upstream call, simulator turn) — unregistering cancels these instead.""" + @cached_property + def state_adapter(self) -> TypeAdapter: + """The rollout's state codec, built only when a state channel is used.""" + return TypeAdapter(type(self.trace.state)) + def adopt(self, task: "asyncio.Task | None") -> None: """Track a handler task serving this session, for cancellation at release. Callers adopt in the same synchronous stretch that fetched the session, so From b9877075842e52d72886cd8ec7a6f92cc49e884f Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:28:48 +0200 Subject: [PATCH 2/3] address review feedback --- tests/v1/test_judges.py | 8 +++- verifiers/v1/envs/agentic_judge/env.py | 8 ++-- verifiers/v1/harnesses/bash/program.py | 39 ++++++++----------- verifiers/v1/harnesses/browser_use/program.py | 31 ++++++++------- verifiers/v1/harnesses/null/program.py | 39 ++++++++----------- verifiers/v1/harnesses/rlm/harness.py | 33 +++++----------- verifiers/v1/judges/rubric.py | 39 +++++++------------ 7 files changed, 84 insertions(+), 113 deletions(-) diff --git a/tests/v1/test_judges.py b/tests/v1/test_judges.py index ab803d2e6e..257cdab566 100644 --- a/tests/v1/test_judges.py +++ b/tests/v1/test_judges.py @@ -481,10 +481,14 @@ def test_rubric_criteria_toml_and_json(tmp_path): toml = rubric_judge(tmp_path).criteria assert [c.name for c in toml] == ["mentions_paris", "is_polite"] assert [c.weight for c in toml] == [3.0, 1.0] - # JSON: both the {"criteria": [...]} object and a bare list parse to the same rubric + # JSON accepts a metadata-bearing object or a bare criteria list. items = [c.model_dump() for c in toml] assert ( - rubric_judge(tmp_path, json.dumps({"criteria": items}), ".json").criteria + rubric_judge( + tmp_path, + json.dumps({"title": "Safety", "criteria": items}), + ".json", + ).criteria == toml ) assert rubric_judge(tmp_path, json.dumps(items), ".json").criteria == toml diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index a71fabb50d..eeb8faf4ae 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -23,8 +23,8 @@ from verifiers.v1.judges.rubric import ( Criterion, RubricVerdicts, - _load_criteria, - _score_verdicts, + load_criteria, + score_verdicts, ) from verifiers.v1.utils.compile import validate_pairing @@ -240,7 +240,7 @@ def build_hint(self) -> str | None: def criteria(self) -> list[Criterion]: if self.rubric is None: return [SOLVED] - return _load_criteria(self.rubric) + return load_criteria(self.rubric) class ScoreConfig(vf.BaseConfig): @@ -324,7 +324,7 @@ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: solution, verdict = by_agent["solver"], by_agent["judge"] verdicts = RubricVerdicts.model_validate(verdict.info.get("verdict")).verdicts criteria = self.config.task.criteria() - scores = _score_verdicts(verdicts, criteria, "the rubric's") + scores = score_verdicts(verdicts, criteria, "the rubric's") for criterion in criteria: solution.record_metric(f"judge/{criterion.name}", scores[criterion.name]) if self.config.score.task_weight != 1.0: diff --git a/verifiers/v1/harnesses/bash/program.py b/verifiers/v1/harnesses/bash/program.py index 35020a512d..643963854f 100644 --- a/verifiers/v1/harnesses/bash/program.py +++ b/verifiers/v1/harnesses/bash/program.py @@ -1,25 +1,18 @@ # /// script # requires-python = ">=3.10" -# dependencies = [ -# "openai", -# "mcp>=1.24.0,<2", -# "httpx", -# "tenacity", -# "pydantic>=2.13.4", -# ] +# dependencies = ["openai", "mcp>=1.24.0,<2", "httpx", "tenacity"] # /// """Secrets arrive through argv so local tool subprocesses do not inherit them.""" import argparse import asyncio +import json import subprocess from contextlib import AsyncExitStack, asynccontextmanager, suppress from pathlib import Path -from typing import Any import httpx from openai import AsyncOpenAI -from pydantic import TypeAdapter, ValidationError from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter SERPER_URL = "https://google.serper.dev/search" @@ -27,9 +20,6 @@ MCP_CALL_ATTEMPTS = 6 MCP_TIMEOUT = 600.0 -JSON_OBJECT_ADAPTER = TypeAdapter(dict[str, Any]) -MESSAGES_ADAPTER = TypeAdapter(list[dict[str, Any]]) - BASH_TOOL = { "type": "function", @@ -331,9 +321,9 @@ async def main() -> None: path = Path(args.initial_messages_file) payload = path.read_bytes() path.unlink() - initial = MESSAGES_ADAPTER.validate_json(payload) + initial = json.loads(payload) client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key) - config = JSON_OBJECT_ADAPTER.validate_json(args.mcp_config or "{}") + config = json.loads(args.mcp_config or "{}") tools = [BASH_TOOL] reserved = {"bash"} if args.edit: @@ -365,19 +355,24 @@ async def main() -> None: for call in message.tool_calls: name = call.function.name try: - tool_args = JSON_OBJECT_ADAPTER.validate_json( - call.function.arguments or "{}" + tool_args = json.loads(call.function.arguments or "{}") + except json.JSONDecodeError as e: + messages.append( + { + "role": "tool", + "tool_call_id": call.id, + "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON", + } ) - except ValidationError as e: + continue + # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls + # below assume a dict, so reject anything else as a tool error rather than crashing. + if not isinstance(tool_args, dict): messages.append( { "role": "tool", "tool_call_id": call.id, - "content": ( - "error: invalid tool arguments " - f"({e.errors(include_url=False)[0]['msg']}); " - "resend as a JSON object" - ), + "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object", } ) continue diff --git a/verifiers/v1/harnesses/browser_use/program.py b/verifiers/v1/harnesses/browser_use/program.py index 1539046e84..1f790c9ded 100644 --- a/verifiers/v1/harnesses/browser_use/program.py +++ b/verifiers/v1/harnesses/browser_use/program.py @@ -6,7 +6,6 @@ # "mcp>=1.24.0,<2", # "httpx", # "tenacity", -# "pydantic>=2.13.4", # ] # /// """A chat loop whose one local tool drives a real Chromium over CDP. @@ -19,6 +18,7 @@ import argparse import asyncio +import json import os import re import shutil @@ -33,15 +33,11 @@ import httpx from openai import AsyncOpenAI -from pydantic import TypeAdapter, ValidationError from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter MCP_CALL_ATTEMPTS = 6 MCP_TIMEOUT = 600.0 -JSON_OBJECT_ADAPTER = TypeAdapter(dict[str, Any]) -MESSAGES_ADAPTER = TypeAdapter(list[dict[str, Any]]) - BROWSER_TOOL_TIMEOUT = 3600 """Matches the bash harness's command timeout.""" @@ -326,7 +322,7 @@ async def main() -> None: path = Path(args.initial_messages_file) payload = path.read_bytes() path.unlink() - initial = MESSAGES_ADAPTER.validate_json(payload) + initial = json.loads(payload) state_dir = Path(args.state_dir) state_dir.mkdir(parents=True, exist_ok=True) endpoint = ( @@ -341,7 +337,7 @@ async def main() -> None: ): endpoint, } client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key) - config = JSON_OBJECT_ADAPTER.validate_json(args.mcp_config or "{}") + config = json.loads(args.mcp_config or "{}") tools = [BROWSER_TOOL] reserved = {"browser"} mcp_tools, dispatch, servers = ( @@ -367,19 +363,24 @@ async def main() -> None: for call in message.tool_calls: name = call.function.name try: - tool_args = JSON_OBJECT_ADAPTER.validate_json( - call.function.arguments or "{}" + tool_args = json.loads(call.function.arguments or "{}") + except json.JSONDecodeError as e: + messages.append( + { + "role": "tool", + "tool_call_id": call.id, + "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON", + } ) - except ValidationError as e: + continue + # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls + # below assume a dict, so reject anything else as a tool error rather than crashing. + if not isinstance(tool_args, dict): messages.append( { "role": "tool", "tool_call_id": call.id, - "content": ( - "error: invalid tool arguments " - f"({e.errors(include_url=False)[0]['msg']}); " - "resend as a JSON object" - ), + "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object", } ) continue diff --git a/verifiers/v1/harnesses/null/program.py b/verifiers/v1/harnesses/null/program.py index efc9fc866d..d5231de072 100644 --- a/verifiers/v1/harnesses/null/program.py +++ b/verifiers/v1/harnesses/null/program.py @@ -1,32 +1,22 @@ # /// script # requires-python = ">=3.11" -# dependencies = [ -# "openai", -# "mcp>=1.24.0,<2", -# "httpx", -# "tenacity", -# "pydantic>=2.13.4", -# ] +# dependencies = ["openai", "mcp>=1.24.0,<2", "httpx", "tenacity"] # /// """The interception endpoint and secret arrive through argv rather than the environment.""" import argparse import asyncio +import json from contextlib import AsyncExitStack, asynccontextmanager, suppress from pathlib import Path -from typing import Any import httpx from openai import AsyncOpenAI -from pydantic import TypeAdapter, ValidationError from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter MCP_CALL_ATTEMPTS = 6 MCP_TIMEOUT = 600.0 -JSON_OBJECT_ADAPTER = TypeAdapter(dict[str, Any]) -MESSAGES_ADAPTER = TypeAdapter(list[dict[str, Any]]) - async def chat( client: AsyncOpenAI, model: str, messages: list[dict], tools: list[dict] @@ -168,13 +158,13 @@ async def main() -> None: path = Path(args.initial_messages_file) payload = path.read_bytes() path.unlink() - initial = MESSAGES_ADAPTER.validate_json(payload) + initial = json.loads(payload) client = AsyncOpenAI( base_url=args.base_url, api_key=args.api_key, timeout=httpx.Timeout(None, connect=5.0), ) - config = JSON_OBJECT_ADAPTER.validate_json(args.mcp_config or "{}") + config = json.loads(args.mcp_config or "{}") if config.get("mcpServers"): # Bound only tool enumeration; each session is opened and closed within this task. async with asyncio.timeout(60): @@ -198,19 +188,24 @@ async def main() -> None: for call in message.tool_calls: name = call.function.name try: - tool_args = JSON_OBJECT_ADAPTER.validate_json( - call.function.arguments or "{}" + tool_args = json.loads(call.function.arguments or "{}") + except json.JSONDecodeError as e: + messages.append( + { + "role": "tool", + "tool_call_id": call.id, + "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON", + } ) - except ValidationError as e: + continue + # Valid JSON can still be a non-object (`[]`, `42`, `null`); the MCP dispatch + # assumes a dict, so reject anything else as a tool error rather than crashing. + if not isinstance(tool_args, dict): messages.append( { "role": "tool", "tool_call_id": call.id, - "content": ( - "error: invalid tool arguments " - f"({e.errors(include_url=False)[0]['msg']}); " - "resend as a JSON object" - ), + "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object", } ) continue diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index b6a3bacd0d..a3d70653fd 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -4,19 +4,9 @@ import logging import random import shlex -from typing import Annotated, Literal, NotRequired - -from pydantic import ( - Field, - FiniteFloat, - OnErrorOmit, - PositiveInt, - Strict, - TypeAdapter, - ValidationError, - model_validator, -) -from typing_extensions import TypedDict +from typing import Literal + +from pydantic import Field, PositiveInt, model_validator from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import HarnessConfig @@ -31,13 +21,6 @@ BuiltinSkill = Literal["edit", "search"] - -class _SessionMeta(TypedDict): - metrics: NotRequired[dict[str, OnErrorOmit[Annotated[FiniteFloat, Strict()]]]] - - -_SESSION_META_ADAPTER = TypeAdapter(_SessionMeta) - RLM_REPO = "github.com/PrimeIntellect-ai/rlm.git" # rlm writes its session under $RLM_HOME/sessions//; point it at a workdir- # relative dir so it stays in the runtime (and is cleaned up with the workdir). @@ -166,7 +149,11 @@ async def rlm(self, runtime: Runtime) -> dict[str, float]: if result.exit_code != 0 or not result.stdout.strip(): return {} try: - meta = _SESSION_META_ADAPTER.validate_json(result.stdout) - except ValidationError: + meta = json.loads(result.stdout) + except json.JSONDecodeError: return {} - return meta.get("metrics", {}) + return { + key: float(value) + for key, value in meta.get("metrics", {}).items() + if isinstance(value, (int, float)) and not isinstance(value, bool) + } diff --git a/verifiers/v1/judges/rubric.py b/verifiers/v1/judges/rubric.py index f41de6dd06..1bfc37c777 100644 --- a/verifiers/v1/judges/rubric.py +++ b/verifiers/v1/judges/rubric.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Annotated, cast -from pydantic import BaseModel, BeforeValidator, Field, TypeAdapter, field_validator +from pydantic import BaseModel, Field, TypeAdapter, field_validator from verifiers.v1.configs.judge import JudgeConfig from verifiers.v1.judge import Judge, JudgeView, judge_question, judge_response @@ -16,7 +16,7 @@ from verifiers.v1.trace import Trace from verifiers.v1.types import ID -_CriterionWeight = Annotated[float, Field(ge=0, allow_inf_nan=False)] +CriterionWeight = Annotated[float, Field(ge=0, allow_inf_nan=False)] RUBRIC_PROMPT = (Path(__file__).resolve().parent / "rubric.txt").read_text( encoding="utf-8" @@ -61,7 +61,7 @@ class Criterion(BaseModel): name: str """Key for the criterion's metric (`/`) and its `weights` override.""" text: str - weight: _CriterionWeight = 1.0 + weight: CriterionWeight = 1.0 """The criterion's share of the reward (overridable per name via `weights` in config).""" choices: list[str] = Field(default_factory=lambda: ["no", "yes"], min_length=2) """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest @@ -75,28 +75,17 @@ def _check_choices(cls, v: list[str]) -> list[str]: return v -_CRITERIA_ADAPTER = TypeAdapter( - Annotated[ - list[Criterion], - BeforeValidator( - lambda value: ( - value.get("criteria", []) if isinstance(value, dict) else value - ) - ), - ] -) +CRITERIA_ADAPTER = TypeAdapter(list[Criterion]) -def _load_criteria( - path: Path, weights: dict[str, _CriterionWeight] | None = None +def load_criteria( + path: Path, weights: dict[str, CriterionWeight] | None = None ) -> list[Criterion]: - """Load either supported rubric shape and apply validated config overrides.""" + """Load a JSON or TOML rubric and apply validated config overrides.""" text = path.read_text(encoding="utf-8") - criteria = ( - _CRITERIA_ADAPTER.validate_python(tomllib.loads(text)) - if path.suffix.lower() == ".toml" - else _CRITERIA_ADAPTER.validate_json(text) - ) + data = tomllib.loads(text) if path.suffix.lower() == ".toml" else json.loads(text) + items = data.get("criteria", []) if isinstance(data, dict) else data + criteria = CRITERIA_ADAPTER.validate_python(items) if not criteria: raise ValueError(f"rubric file '{path}' lists no criteria") names = [criterion.name for criterion in criteria] @@ -127,7 +116,7 @@ class RubricJudgeConfig(JudgeConfig): path: Path """A `.toml` or `.json` file containing a `criteria` list. Relative paths resolve against the evaluation's working directory.""" - weights: dict[str, _CriterionWeight] = Field(default_factory=dict) + weights: dict[str, CriterionWeight] = Field(default_factory=dict) """Per-criterion weight overrides by criterion name (config wins over the file).""" question_field: str = "" """Task field to fill the prompt's `{question}`; empty = the task's prompt rendered as @@ -164,7 +153,7 @@ class RubricVerdicts(BaseModel): verdicts: list[CriterionVerdict] -def _score_verdicts( +def score_verdicts( verdicts: list[CriterionVerdict], criteria: list[Criterion], expected: str, @@ -200,7 +189,7 @@ class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]): @cached_property def criteria(self) -> list[Criterion]: - return _load_criteria(self.config.path, self.config.weights) + return load_criteria(self.config.path, self.config.weights) async def grade_batch( self, task: TaskData, trace: Trace, batch: list[Criterion] @@ -249,7 +238,7 @@ def render(c: Criterion) -> str: ) verdicts = RubricVerdicts.model_validate(obj).verdicts # A malformed verdict is a judge failure and must error the rollout, not score the model. - return _score_verdicts(verdicts, batch, "the batch's") + return score_verdicts(verdicts, batch, "the batch's") async def score(self, task: TaskData, trace: Trace) -> float: criteria = self.criteria From 04a23901035e318b7eb48e89ac6bae4a8c5a884b Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:58:09 +0200 Subject: [PATCH 3/3] refactor(v1): use public adapter names --- verifiers/v1/cli/output.py | 6 +++--- verifiers/v1/legacy.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py index e6da861811..3c184a16a6 100644 --- a/verifiers/v1/cli/output.py +++ b/verifiers/v1/cli/output.py @@ -31,7 +31,7 @@ """Filename a run's resolved config is written to (re-runnable via `@ config.toml`).""" # Compiling an adapter is the expensive part; run output reuses only a few model classes. -_type_adapter = cache(TypeAdapter) +type_adapter = cache(TypeAdapter) def output_path(config: EvalConfig) -> Path: @@ -76,7 +76,7 @@ def save_config(config: BaseModel, results_dir: Path) -> None: def write_episode(results_dir: Path, episode: Episode) -> None: """Serialize and append one rollout episode in the worker thread.""" # Preserve fields declared by typed Trace subclasses nested in the episode. - data = _type_adapter(type(episode)).dump_json(episode, exclude_none=True) + data = type_adapter(type(episode)).dump_json(episode, exclude_none=True) with (results_dir / TRACES_FILE).open("ab") as f: f.write(data + b"\n") @@ -92,7 +92,7 @@ def read_episodes(results_dir: Path, trace_type: type) -> list[Episode]: `trace_type` (`Trace[WireTaskData, ...]` reads any taskset's file without importing it). A pre-episode line (one bare trace) is wrapped as a single-trace record, so both file generations read uniformly.""" - trace_adapter = _type_adapter(trace_type) + trace_adapter = type_adapter(trace_type) episodes: list[Episode] = [] with (results_dir / TRACES_FILE).open(encoding="utf-8") as f: for line in f: diff --git a/verifiers/v1/legacy.py b/verifiers/v1/legacy.py index abaea20bad..0dd81f02cb 100644 --- a/verifiers/v1/legacy.py +++ b/verifiers/v1/legacy.py @@ -76,7 +76,7 @@ def _as_dict(obj: Any) -> Any: return obj -_TOOLS_ADAPTER = TypeAdapter( +TOOLS_ADAPTER = TypeAdapter( list[OnErrorOmit[Annotated[Tool, BeforeValidator(_as_dict)]]] ) @@ -86,7 +86,7 @@ def _to_v1_tools(raw: Any) -> list[Tool] | None: shapes are identical (name/description/parameters/strict), so this is a re-validation; malformed entries are dropped rather than failing the whole trace mapping.""" try: - defs = _TOOLS_ADAPTER.validate_python(raw or []) + defs = TOOLS_ADAPTER.validate_python(raw or []) except ValidationError: return None return defs or None