diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index f0c294f764..6e4017312a 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -367,7 +367,7 @@ async def test_multi_agent_env(run_v1, tmp_path): # agent info (completion order — the gathered seats land in either order). (line,) = (tmp_path / "traces.jsonl").read_text().splitlines() row = json.loads(line) - assert row["env"] == "duet-v1" + assert row["env"] == {"id": "duet-v1"} by_name = {t["agent"]["name"]: t for t in row["traces"]} assert set(by_name) == {"a", "b"} assert by_name["a"]["agent"]["trainable"] is True diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 7af9b7f140..cacbd5aca8 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -115,10 +115,10 @@ from verifiers.v1.trace import ( TRACE_VERSION, AgentInfo, + AgentSpan, Branch, Error, EvalRunInfo, - GenerationSpan, ModelCall, Reward, RunInfo, @@ -144,7 +144,6 @@ Response, Sampling, SamplingConfig, - StrictBaseModel, SystemMessage, TextContentPart, Tool, @@ -177,7 +176,6 @@ "Response", "Sampling", "SamplingConfig", - "StrictBaseModel", "SystemMessage", "TextContentPart", "Tool", @@ -213,7 +211,7 @@ "Timing", "TimeSpan", "TimeSplit", - "GenerationSpan", + "AgentSpan", "Error", # decorators "stop", diff --git a/verifiers/v1/artifacts.py b/verifiers/v1/artifacts.py index 1d60e6ae1d..3ce281366a 100644 --- a/verifiers/v1/artifacts.py +++ b/verifiers/v1/artifacts.py @@ -8,9 +8,7 @@ from pathlib import PurePosixPath from typing import TYPE_CHECKING -from pydantic import Field - -from verifiers.v1.types import StrictBaseModel +from pydantic import BaseModel, Field if TYPE_CHECKING: from verifiers.v1.runtimes import Runtime @@ -25,7 +23,7 @@ agent's image, so the repo is already there and only its output has to travel.""" -class Artifact(StrictBaseModel): +class Artifact(BaseModel): """One path to restore at the same location in another runtime.""" source: str diff --git a/verifiers/v1/cli/dashboard/eval.py b/verifiers/v1/cli/dashboard/eval.py index 68e1b39cc1..169dcb4760 100644 --- a/verifiers/v1/cli/dashboard/eval.py +++ b/verifiers/v1/cli/dashboard/eval.py @@ -393,18 +393,19 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None: phase_count: dict[str, int] = {} model_secs = harness_secs = 0.0 for trace in done: - prompt, completion, cached, reasoning, _ = _tokens(trace) - total_in += prompt - total_out += completion - if cached is not None: - total_cached += cached - have_cached = True - if reasoning is not None: - total_reasoning += reasoning - have_reasoning = True - if trace.usage is not None and trace.usage.cost is not None: - total_cost += trace.usage.cost - have_cost = True + total_in += trace.num_input_tokens + total_out += trace.num_output_tokens + usage = trace.usage + if usage is not None: + if usage.cached_input_tokens is not None: + total_cached += usage.cached_input_tokens + have_cached = True + if usage.reasoning_tokens is not None: + total_reasoning += usage.reasoning_tokens + have_reasoning = True + if usage.cost is not None: + total_cost += usage.cost + have_cost = True # Judge / auxiliary scoring calls (off the message graph) shown separately from the agent's. judge = Usage.aggregate(trace.extra_usage) if judge is not None: @@ -413,13 +414,13 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None: if judge.cost is not None: total_judge_cost += judge.cost have_judge = True - for phase in ("boot", "setup", "generation", "finalize", "scoring"): + for phase in ("boot", "setup", "agent", "finalize", "scoring"): span = getattr(trace.timing, phase) if span.end: # phase was timed for this rollout phase_secs[phase] = phase_secs.get(phase, 0.0) + span.duration phase_count[phase] = phase_count.get(phase, 0) + 1 - model_secs += trace.timing.generation.model.duration - harness_secs += trace.timing.generation.harness.duration + model_secs += trace.timing.agent.model.duration + harness_secs += trace.timing.agent.harness.duration if ( total_in or total_out @@ -448,12 +449,12 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None: usage.append(cost) grid.add_row("usage", " · ".join(usage)) time_segments = [] - for phase in ("boot", "setup", "generation", "finalize", "scoring"): + for phase in ("boot", "setup", "agent", "finalize", "scoring"): count = phase_count.get(phase) if not count: continue segment = f"{phase} {format_time(phase_secs[phase] / count)}" - if phase == "generation": + if phase == "agent": segment += ( f" (model {format_time(model_secs / count)}" f" + harness {format_time(harness_secs / count)})" @@ -464,26 +465,6 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None: return grid if grid.row_count else None -def _tokens(trace: Trace) -> tuple[int, int, int | None, int | None, int]: - """Input/output tokens summed across all branches: per branch, output is every assistant - (completion) token generated across its turns and input is the fed-in tokens counted once - (system + user + tool) — the final sequence minus everything the model generated. A rollout - yields one training sample per branch (a linear trace is a single branch; compaction and - subagents add more), so the totals sum them — matching `Trace.num_input_tokens` / - `Trace.num_output_tokens`, whose sum is `num_total_tokens`. - - Both counts come from provider-reported usage. Returns the branch count from the same derived - view so each dashboard tick materializes it once.""" - usage = trace.usage - cached = usage.cached_input_tokens if usage else None - reasoning = usage.reasoning_tokens if usage else None - branches = trace.branches - nbranches = len(branches) - prompt = sum(b.num_input_tokens for b in branches) - completion = sum(b.num_output_tokens for b in branches) - return prompt, completion, cached, reasoning, nbranches - - def _stage(trace: Trace) -> str: """The stage a live (not-yet-done) rollout is in, derived from its trace's timing spans — the engine opens and closes each span exactly at the stage transitions, so @@ -495,7 +476,7 @@ def _stage(trace: Trace) -> str: for stage, span in ( ("scoring", trace.timing.scoring), ("finalize", trace.timing.finalize), - ("running", trace.timing.generation), + ("running", trace.timing.agent), ("setup", trace.timing.setup), ("boot", trace.timing.boot), ): @@ -551,7 +532,9 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table: base = f"name={task.name[:32]}" if task.name else f"idx={task.idx}" if not slot.traces: if slot.done: # the env's rollout() itself failed before any trace - error = slot.episode.error if slot.episode is not None else None + error = ( + slot.episode.last_error if slot.episode is not None else None + ) group_rows.append( ( "error", @@ -602,7 +585,7 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table: end = ( t.timing.scoring.end or t.timing.finalize.end - or t.timing.generation.end + or t.timing.agent.end # a rollout that errored in boot/setup has only that span's end — freeze there # once done, else (still running) the timer would grow off `now` forever or ( @@ -612,8 +595,12 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table: ) or now ) - prompt, completion, cached, reasoning, nbranches = _tokens(t) - cost = t.usage.cost if t.usage else None + prompt, completion = t.num_input_tokens, t.num_output_tokens + nbranches = t.num_branches + usage = t.usage + cached = usage.cached_input_tokens if usage else None + reasoning = usage.reasoning_tokens if usage else None + cost = usage.cost if usage else None tokens = "" if prompt or completion: tokens = f"{format_count(prompt)}/{format_count(completion)} tokens" diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py index 11c9cbb024..27a947ada8 100644 --- a/verifiers/v1/cli/debug.py +++ b/verifiers/v1/cli/debug.py @@ -123,16 +123,16 @@ def record_debug_error( action_timeout: float | None, ) -> None: now = time.time() - for span in (trace.timing.boot, trace.timing.setup, trace.timing.generation): + for span in (trace.timing.boot, trace.timing.setup, trace.timing.agent): if span.start and not span.end: span.end = now - in_action = bool(trace.timing.generation.start) + in_action = bool(trace.timing.agent.start) stage = ( "debug action" if in_action else "setup" if trace.timing.setup.start else "boot" ) timeout = action_timeout if in_action else setup_timeout error_start = ( - trace.timing.generation.start + trace.timing.agent.start if in_action else trace.timing.setup.start or trace.timing.boot.start ) @@ -234,9 +234,9 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]: await runtime.prepare_execution([]) trace.timing.setup.end = time.time() - trace.timing.generation.start = time.time() + trace.timing.agent.start = time.time() debug.update(await run_action(runtime, config)) - trace.timing.generation.end = time.time() + trace.timing.agent.end = time.time() if not debug.get("ok"): record_action_failure(trace, debug) trace.stop(str(debug["reason"])) @@ -246,7 +246,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]: except Exception as e: # noqa: BLE001 - persist any framework failure on the trace record_debug_error(trace, debug, e, setup_timeout, config.timeout.total) finally: - trace.split_generation() + trace.split_agent_time() trace.info["debug"] = debug try: await runtime.stop() diff --git a/verifiers/v1/cli/replay.py b/verifiers/v1/cli/replay.py index df0e9130ad..9d4d11d610 100644 --- a/verifiers/v1/cli/replay.py +++ b/verifiers/v1/cli/replay.py @@ -84,7 +84,7 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac episodes = read_episodes( source, Trace[WireTaskData, state_cls(task_cls), WireAgentConfig] ) - sourced = [(trace, e.env) for e in episodes for trace in e.traces] + sourced = [(trace, e.env.id) for e in episodes for trace in e.traces] if config.num_traces is not None: sourced = sourced[: config.num_traces] traces = [trace for trace, _ in sourced] diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index b18a916647..e3de1420f8 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -20,7 +20,7 @@ _declared_agent_configs, default_agent_harness, ) -from verifiers.v1.episode import Episode +from verifiers.v1.episode import EnvInfo, Episode from verifiers.v1.errors import EnvError, boundary from verifiers.v1.harness import Harness, HarnessConfig from verifiers.v1.interception import ( @@ -257,7 +257,7 @@ async def run_episode( completed subset, its exception on the episode's `errors`. `on_trace` observes each agent-run's trace at mint; `on_discard` its abandonment (a per-agent retry mints a replacement).""" - episode = Episode(env=self.config.env_id) + episode = Episode(env=EnvInfo(id=self.config.env_id)) agents = self._episode_agents(ctx, episode.traces, on_trace, on_discard) try: async with asyncio.timeout(self.config.timeout.episode): diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index a4b5472557..b71b9e0357 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -19,10 +19,9 @@ import tomllib from pathlib import Path -from pydantic import Field, field_validator +from pydantic import BaseModel, Field, field_validator import verifiers.v1 as vf -from verifiers.v1.types import StrictBaseModel from verifiers.v1.utils.compile import validate_pairing VERDICT_FILE = "/tmp/verdict.json" @@ -39,7 +38,7 @@ {prompt}""" -class Criterion(StrictBaseModel): +class Criterion(BaseModel): """One rubric criterion — the plugged rubric judge's format, mirrored so the same `criteria` files grade both judges.""" diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index ccce076402..7e8c54bdb9 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -3,51 +3,79 @@ import uuid from typing import Generic -from pydantic import Field +from pydantic import BaseModel, Field from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData from verifiers.v1.trace import AgentConfigT, Error, Trace -from verifiers.v1.types import StrictBaseModel +from verifiers.v1.types import Usage -class Episode(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]): - """One run of a task, whole: its identity and standing (`id`, `env`, `errors`) - next to its flat `traces` — the object `finalize()` receives, the engine - returns, and the durability envelope: one episode is one `traces.jsonl` line - and one serve reply, so it persists and arrives whole or not at all — a torn - line is the whole episode owed again, and a failure before any trace minted - still leaves its errors here. Episode standing lives ONLY here (zero - redundancy on the traces); per-trace facts (`agent`, per-trace errors) stay - on the traces, which remain the atomic unit. +class EnvInfo(BaseModel): + """The env that ran the episode, self-describing without the run's config.""" - `errors` are failures not attributable to any one trace (the env's - `run`/`finalize` hooks, plus prior attempts' when retried). + id: str = "" + """`EnvConfig.env_id`, e.g. `agentic-judge+gsm8k-v1`.""" - The type parameters serve the wire loaders: `WireEpisode` reads any taskset's - episodes without importing the taskset.""" + +class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]): + """The artifact Env.run produces. Contains multiple agents' traces.""" id: str = Field(default_factory=lambda: uuid.uuid4().hex) - env: str = "" - """The env that ran the episode (`EnvConfig.env_id`, e.g. - `agentic-judge+gsm8k-v1`).""" + + env: EnvInfo = Field(default_factory=EnvInfo) + """The env that produced this episode.""" ok: bool = False - """THE success sentinel — the resume unit's keep-verdict, stamped by the - engine when the final attempt's hooks and every trace concluded clean. - Distinct from `errors` emptiness: a retried-and-recovered episode is `ok` - and still keeps its earlier attempts' errors.""" + """Whether the episode completed successfully.""" errors: list[Error] = Field(default_factory=list) + """Every error captured across attempts, oldest to newest.""" traces: list[Trace[DataT, StateT, AgentConfigT]] = Field(default_factory=list) + """Every agent's trace, in completion order.""" @property - def error(self) -> Error | None: + def last_error(self) -> Error | None: + """The last episode-level error captured across attempts.""" return self.errors[-1] if self.errors else None + @property + def usage(self) -> Usage | None: + """Provider-reported usage summed across every trace's model calls; + judge/off-graph usage stays on the traces (`Trace.extra_usage`).""" + return Usage.aggregate(u for t in self.traces if (u := t.usage) is not None) + + @property + def num_input_tokens(self) -> int: + """Fed-in tokens (system + user + tool), summed across traces.""" + return sum(t.num_input_tokens for t in self.traces) + + @property + def num_output_tokens(self) -> int: + """Model-generated tokens across all turns, summed across traces.""" + return sum(t.num_output_tokens for t in self.traces) + + @property + def num_total_tokens(self) -> int: + """Final sequence lengths per branch, summed across traces.""" + return sum(t.num_total_tokens for t in self.traces) + + @property + def num_turns(self) -> int: + """Sampled turns, summed across traces.""" + return sum(t.num_turns for t in self.traces) + + @property + def by_agent(self) -> dict[str, list[Trace[DataT, StateT, AgentConfigT]]]: + """Traces grouped by agent name (e.g. n solvers), in completion order.""" + grouped: dict[str, list[Trace[DataT, StateT, AgentConfigT]]] = {} + for trace in self.traces: + grouped.setdefault(trace.agent.name, []).append(trace) + return grouped + @classmethod def of(cls, trace: Trace, env: str = "") -> "Episode": """The single-agent record: one trace as its own episode.""" - return cls(env=env, traces=[trace], ok=trace.ok) + return cls(env=EnvInfo(id=env), traces=[trace], ok=trace.ok) WireEpisode = Episode[WireTaskData, State, WireAgentConfig] diff --git a/verifiers/v1/graph.py b/verifiers/v1/graph.py index 34ed992840..f81480d43b 100644 --- a/verifiers/v1/graph.py +++ b/verifiers/v1/graph.py @@ -25,7 +25,7 @@ from typing import TYPE_CHECKING, Any import numpy as np -from pydantic import ConfigDict, Field, field_serializer, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator from pydantic.json_schema import SkipJsonSchema from renderers.base import MultiModalData, PlaceholderRange, RenderedTokens @@ -34,7 +34,6 @@ KeptTokens, Message, Response, - StrictBaseModel, TextContentPart, Tool, ToolMessage, @@ -62,7 +61,7 @@ def _decode_ndarray(d: dict) -> np.ndarray: return np.frombuffer(d["data"], dtype=np.dtype(d["dtype"])).reshape(d["shape"]) -class MessageNode(StrictBaseModel): +class MessageNode(BaseModel): """One message in the graph: a message plus the tokens it adds to the cumulative sequence. Concatenating a root→leaf path's nodes reconstructs that branch's full token sequence; the mask/logprobs make it a training sample.""" @@ -121,7 +120,7 @@ class MessageNode(StrictBaseModel): sampling-replay training. Rides the wire as raw-bytes `__nd__` dicts; kept off disk by the dump-site `exclude` in prime-rl.""" - model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + model_config = ConfigDict(arbitrary_types_allowed=True) @field_serializer("multi_modal_data") def serialize_multi_modal_data(self, mmd: MultiModalData | None) -> dict | None: diff --git a/verifiers/v1/judge.py b/verifiers/v1/judge.py index c1cdd81d0a..308c971fc8 100644 --- a/verifiers/v1/judge.py +++ b/verifiers/v1/judge.py @@ -62,7 +62,7 @@ async def correct(self, trace) -> float: ) from verifiers.v1.dialects.chat import message_to_wire from verifiers.v1.scoring import parse_judge_choice -from verifiers.v1.types import Messages, StrictBaseModel, Usage +from verifiers.v1.types import Messages, Usage from verifiers.v1.utils.generic import concrete_type if TYPE_CHECKING: @@ -72,7 +72,7 @@ async def correct(self, trace) -> float: ParsedT = TypeVar("ParsedT") -class JudgeResponse(StrictBaseModel, Generic[ParsedT]): +class JudgeResponse(BaseModel, Generic[ParsedT]): text: str parsed: ParsedT | None = None usage: Usage | None = None @@ -197,8 +197,7 @@ async def complete( ) if response.parsed is None: raise RuntimeError( - f"judge returned no parseable structured output " - f"(finish_reason={choice.finish_reason})" + f"judge returned no parseable structured output (finish_reason={choice.finish_reason})" ) else: completion = await client.chat.completions.create(**kwargs) diff --git a/verifiers/v1/judges/rubric.py b/verifiers/v1/judges/rubric.py index 0419454846..35dadae665 100644 --- a/verifiers/v1/judges/rubric.py +++ b/verifiers/v1/judges/rubric.py @@ -9,13 +9,13 @@ from pathlib import Path from typing import cast -from pydantic import Field, field_validator +from pydantic import BaseModel, Field, field_validator from verifiers.v1.configs.judge import JudgeConfig from verifiers.v1.judge import Judge, JudgeView, judge_question, judge_response from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace -from verifiers.v1.types import ID, StrictBaseModel +from verifiers.v1.types import ID RUBRIC_PROMPT = (Path(__file__).resolve().parent / "rubric.txt").read_text( encoding="utf-8" @@ -56,7 +56,7 @@ def first_verdicts_object(text: str) -> dict | None: return None -class Criterion(StrictBaseModel): +class Criterion(BaseModel): name: str """Key for the criterion's metric (`/`) and its `weights` override.""" text: str @@ -109,13 +109,13 @@ class RubricJudgeConfig(JudgeConfig): handle either. Transient HTTP failures are already retried by the OpenAI client.""" -class CriterionVerdict(StrictBaseModel): +class CriterionVerdict(BaseModel): name: str reason: str verdict: str -class RubricVerdicts(StrictBaseModel): +class RubricVerdicts(BaseModel): verdicts: list[CriterionVerdict] diff --git a/verifiers/v1/legacy.py b/verifiers/v1/legacy.py index be36b3a919..d964bae322 100644 --- a/verifiers/v1/legacy.py +++ b/verifiers/v1/legacy.py @@ -36,8 +36,8 @@ from verifiers.v1.task import WireTaskData from verifiers.v1.trace import ( AgentInfo, + AgentSpan, Error, - GenerationSpan, ModelCall, Reward, TimeSpan, @@ -199,7 +199,8 @@ def _to_v1_tokens(raw: Any) -> TurnTokens | None: def _timing(raw: Any) -> Timing: """Map the v0 timing record's generation/scoring durations onto a v1 ``Timing`` (we only have durations, so each span is encoded as start=0, end=duration). - v0's per-turn ``model``/``env`` span collections carry the generation split.""" + v0's ``generation`` duration becomes the agent span; its per-turn ``model``/``env`` + span collections carry the model/harness split.""" def _dur(node: Any) -> float: if isinstance(node, dict): @@ -212,7 +213,7 @@ def _dur(node: Any) -> float: raw = raw or {} return Timing( - generation=GenerationSpan( + agent=AgentSpan( start=0.0, end=_dur(raw.get("generation")), model=TimeSplit(duration=_dur(raw.get("model"))), diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index b59b62096c..3544c129f8 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -297,7 +297,7 @@ async def open(self) -> bool: raise now = time.time() self.trace.timing.setup.end = now - self.trace.timing.generation.start = now + self.trace.timing.agent.start = now return True async def step(self, messages: Messages | None = None) -> bool: @@ -397,8 +397,8 @@ async def close(self) -> Trace: try: await self._stack.aclose() finally: - if trace.timing.generation.start and not trace.timing.generation.end: - trace.timing.generation.end = time.time() + if trace.timing.agent.start and not trace.timing.agent.end: + trace.timing.agent.end = time.time() if not self._failed and self._opened: trace.timing.finalize.start = time.time() async with boundary(TaskError, "task finalize"): @@ -430,13 +430,13 @@ async def close(self) -> Trace: for span in ( trace.timing.boot, trace.timing.setup, - trace.timing.generation, + trace.timing.agent, trace.timing.finalize, trace.timing.scoring, ): if span.start and not span.end: span.end = now - trace.split_generation() + trace.split_agent_time() if runtime is not None: try: await self.harness.cleanup(trace, runtime) diff --git a/verifiers/v1/state.py b/verifiers/v1/state.py index 8cd84c6f0c..ba7cec096f 100644 --- a/verifiers/v1/state.py +++ b/verifiers/v1/state.py @@ -4,14 +4,13 @@ from serialized traces. """ -from pydantic import ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field from typing_extensions import TypeVar -from verifiers.v1.types import StrictBaseModel from verifiers.v1.utils.generic import concrete_type -class State(StrictBaseModel): +class State(BaseModel): model_config = ConfigDict(ser_json_inf_nan="constants") artifacts: dict[str, bytes] = Field(default_factory=dict) diff --git a/verifiers/v1/task.py b/verifiers/v1/task.py index 2786be0cc0..acb25b1a30 100644 --- a/verifiers/v1/task.py +++ b/verifiers/v1/task.py @@ -30,7 +30,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, ClassVar, Generic, Self -from pydantic import ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field from pydantic_config import BaseConfig from typing_extensions import TypeVar @@ -39,7 +39,7 @@ from verifiers.v1.decorators import discover_decorated, invoke_all from verifiers.v1.errors import TaskError, boundary from verifiers.v1.state import StateT -from verifiers.v1.types import Messages, StrictBaseModel, content_text +from verifiers.v1.types import Messages, content_text from verifiers.v1.utils.generic import concrete_type if TYPE_CHECKING: @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) -class TaskResources(StrictBaseModel): +class TaskResources(BaseModel): model_config = ConfigDict(frozen=True) cpu: float | None = None @@ -64,7 +64,7 @@ class TaskResources(StrictBaseModel): """Disk in GB (enforced by prime; advisory on docker/modal).""" -class TaskTimeout(StrictBaseModel): +class TaskTimeout(BaseModel): """Optional per-task timeout overrides, in seconds.""" model_config = ConfigDict(frozen=True) @@ -75,7 +75,7 @@ class TaskTimeout(StrictBaseModel): scoring: float | None = None -class TaskData(StrictBaseModel): +class TaskData(BaseModel): """The task's wire half: one row's pure data, a frozen pydantic model. Subclass per dataset to add typed task-specific fields next to the base fields; behavior lives on `Task`, which wraps this (`self.data`).""" diff --git a/verifiers/v1/tasksets/harbor/taskset.py b/verifiers/v1/tasksets/harbor/taskset.py index a3d00d1af6..d4bca5732b 100644 --- a/verifiers/v1/tasksets/harbor/taskset.py +++ b/verifiers/v1/tasksets/harbor/taskset.py @@ -22,7 +22,7 @@ from functools import lru_cache from pathlib import Path -from pydantic import Field +from pydantic import BaseModel, Field from verifiers.v1.artifacts import Artifact, collect from verifiers.v1.configs.taskset import TasksetConfig @@ -32,7 +32,6 @@ from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout from verifiers.v1.taskset import Taskset from verifiers.v1.trace import Trace -from verifiers.v1.types import StrictBaseModel CACHE = Path.home() / ".cache" / "harbor" HARBOR_INSTALL_HINT = "uv sync --python 3.12 --extra harbor" @@ -73,12 +72,12 @@ class HarborConfig(TasksetConfig): has what the task needs (e.g. you've pointed the runtime at the right image).""" -class Author(StrictBaseModel): +class Author(BaseModel): name: str | None = None email: str | None = None -class CollectHook(StrictBaseModel): +class CollectHook(BaseModel): """One `[[verifier.collect]]` command, run in the agent's box by `finalize`.""" command: str diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 0ee0b792f1..6fbf25e403 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Annotated, Any, Generic, Literal import numpy as np -from pydantic import Field, PrivateAttr +from pydantic import BaseModel, Field, PrivateAttr from renderers.base import MultiModalData from typing_extensions import TypeVar @@ -27,7 +27,6 @@ KeptTokens, Messages, Sampling, - StrictBaseModel, Tool, ToolMessage, Usage, @@ -50,7 +49,7 @@ """Raw tensor fields kept on the msgpack wire but excluded from disk serialization.""" -class TimeSpan(StrictBaseModel): +class TimeSpan(BaseModel): """Wall-clock timestamps with a derived, non-serialized duration in seconds.""" start: float = 0.0 @@ -61,34 +60,34 @@ def duration(self) -> float: return max(0.0, self.end - self.start) if self.end else 0.0 -class TimeSplit(StrictBaseModel): +class TimeSplit(BaseModel): """Records a measured duration in seconds.""" duration: float = 0.0 -class GenerationSpan(TimeSpan): +class AgentSpan(TimeSpan): model: TimeSplit = Field(default_factory=TimeSplit) harness: TimeSplit = Field(default_factory=TimeSplit) -class Timing(StrictBaseModel): +class Timing(BaseModel): start: float = Field(default_factory=time.time) boot: TimeSpan = Field(default_factory=TimeSpan) setup: TimeSpan = Field(default_factory=TimeSpan) - generation: GenerationSpan = Field(default_factory=GenerationSpan) + agent: AgentSpan = Field(default_factory=AgentSpan) finalize: TimeSpan = Field(default_factory=TimeSpan) scoring: TimeSpan = Field(default_factory=TimeSpan) -class Error(StrictBaseModel): +class Error(BaseModel): type: str message: str status_code: int | None = None traceback: str | None = None -class VersionInfo(StrictBaseModel): +class VersionInfo(BaseModel): version: str commit: str | None = None @@ -104,7 +103,7 @@ def _current_build() -> VersionInfo: AgentConfigT = TypeVar("AgentConfigT", bound=AgentConfig, default=AgentConfig) -class AgentInfo(StrictBaseModel, Generic[AgentConfigT]): +class AgentInfo(BaseModel, Generic[AgentConfigT]): config: AgentConfigT """The resolved config that rebuilds the agent (`Agent(trace.agent.config)`).""" runtime: RuntimeInfo | None = None @@ -115,7 +114,7 @@ class AgentInfo(StrictBaseModel, Generic[AgentConfigT]): """Whether this trace's tokens train the run's policy.""" -class TraceTask(StrictBaseModel, Generic[DataT]): +class TraceTask(BaseModel, Generic[DataT]): """The task as recorded on the trace, self-describing without the run's config.""" type: str @@ -124,7 +123,7 @@ class TraceTask(StrictBaseModel, Generic[DataT]): """The (immutable) row being solved.""" -class Reward(StrictBaseModel): +class Reward(BaseModel): score: float weight: float = 1.0 @@ -133,14 +132,14 @@ def value(self) -> float: return self.score * self.weight -class EvalRunInfo(StrictBaseModel): +class EvalRunInfo(BaseModel): type: Literal["eval"] = "eval" id: str step: int | None = None -class TrainRunInfo(StrictBaseModel): +class TrainRunInfo(BaseModel): type: Literal["train"] = "train" id: str @@ -151,7 +150,7 @@ class TrainRunInfo(StrictBaseModel): """The run a trace belongs to, discriminated on `type`.""" -class ModelCall(StrictBaseModel): +class ModelCall(BaseModel): """A model call, automatically recorded at intercept time.""" node: int | None = None @@ -172,7 +171,7 @@ class ModelCall(StrictBaseModel): """The failure that ended this call, coupled to the exchange that caused it.""" -class Branch(StrictBaseModel): +class Branch(BaseModel): """A root-to-leaf graph path; each branch becomes one training sample.""" index: int @@ -298,7 +297,7 @@ def num_input_tokens(self) -> int: return self.num_total_tokens - self.num_output_tokens -class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]): +class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): version: int = TRACE_VERSION """The trace schema this trace serializes as.""" id: str = Field(default_factory=lambda: uuid.uuid4().hex) @@ -492,14 +491,14 @@ def stop(self, condition: str) -> None: if self.stop_condition is None: self.stop_condition = condition - def split_generation(self) -> None: - """Split the generation span into model and harness time.""" - gen = self.timing.generation - if not gen.end: + def split_agent_time(self) -> None: + """Split the agent span into model and harness time.""" + span = self.timing.agent + if not span.end: return model = sum(call.time.duration for call in self.calls) - gen.model.duration = min(model, gen.duration) - gen.harness.duration = gen.duration - gen.model.duration + span.model.duration = min(model, span.duration) + span.harness.duration = span.duration - span.model.duration def record_error(self, error: Exception) -> None: """Record an error, and stop the trace as failed.""" diff --git a/verifiers/v1/types.py b/verifiers/v1/types.py index 7ee25f8502..dcb99f7067 100644 --- a/verifiers/v1/types.py +++ b/verifiers/v1/types.py @@ -7,20 +7,16 @@ from typing_extensions import TypedDict -class StrictBaseModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - -class TextContentPart(StrictBaseModel): +class TextContentPart(BaseModel): type: Literal["text"] = "text" text: str -class ImageUrlSource(StrictBaseModel): +class ImageUrlSource(BaseModel): url: str -class ImageUrlContentPart(StrictBaseModel): +class ImageUrlContentPart(BaseModel): type: Literal["image_url"] = "image_url" image_url: ImageUrlSource @@ -57,24 +53,24 @@ def content_text(content: "MessageContent | None") -> str: ) -class SystemMessage(StrictBaseModel): +class SystemMessage(BaseModel): role: Literal["system"] = "system" content: MessageContent -class UserMessage(StrictBaseModel): +class UserMessage(BaseModel): role: Literal["user"] = "user" content: MessageContent -class ToolCall(StrictBaseModel): +class ToolCall(BaseModel): id: str name: str arguments: str """Raw JSON string of arguments, exactly as the model emitted it.""" -class AssistantMessage(StrictBaseModel): +class AssistantMessage(BaseModel): role: Literal["assistant"] = "assistant" content: str | None = None reasoning_content: str | None = None @@ -83,7 +79,7 @@ class AssistantMessage(StrictBaseModel): """Opaque native items replayed to preserve signed or encrypted reasoning state.""" -class ToolMessage(StrictBaseModel): +class ToolMessage(BaseModel): role: Literal["tool"] = "tool" tool_call_id: str content: MessageContent @@ -98,7 +94,7 @@ class ToolMessage(StrictBaseModel): Messages = list[Message] -class Tool(StrictBaseModel): +class Tool(BaseModel): name: str description: str parameters: dict[str, Any] @@ -108,7 +104,7 @@ class Tool(StrictBaseModel): FinishReason = Literal["stop", "length", "tool_calls"] | None -class Usage(StrictBaseModel): +class Usage(BaseModel): """Provider token accounting. `prompt_tokens` excludes cache reads; `input_tokens` adds them back. Reasoning tokens @@ -191,10 +187,10 @@ class KeptTokens: counts: Any -class TurnTokens(StrictBaseModel): +class TurnTokens(BaseModel): """Training tokens from renderer tokenization or provider-returned token IDs.""" - model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + model_config = ConfigDict(arbitrary_types_allowed=True) prompt_ids: list[int] = Field(default_factory=list) completion_ids: list[int] = Field(default_factory=list) @@ -220,7 +216,7 @@ class TurnTokens(StrictBaseModel): kept_tokens: KeptTokens | None = Field(default=None, exclude=True) -class Response(StrictBaseModel): +class Response(BaseModel): id: str created: int model: str