diff --git a/docs/v0/development.md b/docs/v0/development.md index 8414db5c6a..e4e5957e7d 100644 --- a/docs/v0/development.md +++ b/docs/v0/development.md @@ -263,26 +263,22 @@ prime eval run my-environment -m openai/gpt-4.1-mini -n 5 # my_environment.py import verifiers as vf + def load_environment(**kwargs): """Load the environment.""" dataset = vf.load_example_dataset("dataset_name") parser = vf.XMLParser(fields=["reasoning", "answer"]) - + def reward_func(parser, completion, answer, **kwargs): return 1.0 if parser.parse_answer(completion) == answer else 0.0 - + rubric = vf.Rubric( funcs=[reward_func, parser.get_format_reward_func()], weights=[1.0, 0.2], - parser=parser - ) - - return vf.SingleTurnEnv( - dataset=dataset, parser=parser, - rubric=rubric, - **kwargs ) + + return vf.SingleTurnEnv(dataset=dataset, parser=parser, rubric=rubric, **kwargs) ``` ## Quick Reference diff --git a/docs/v0/environments.md b/docs/v0/environments.md index 3598540d06..a105d388d7 100644 --- a/docs/v0/environments.md +++ b/docs/v0/environments.md @@ -47,12 +47,15 @@ The simplest single-turn environments need only a dataset of tasks and a reward import verifiers as vf from datasets import Dataset + def load_environment(): # Your task data - dataset = Dataset.from_list([ - {"prompt": [{"role": "user", "content": "What is 2+2?"}], "answer": "4"}, - {"prompt": [{"role": "user", "content": "What is 3*5?"}], "answer": "15"}, - ]) + dataset = Dataset.from_list( + [ + {"prompt": [{"role": "user", "content": "What is 2+2?"}], "answer": "4"}, + {"prompt": [{"role": "user", "content": "What is 3*5?"}], "answer": "15"}, + ] + ) # Your reward function async def correct_answer(completion, answer) -> float: @@ -84,10 +87,12 @@ Depending on what your environment needs, you can include `answer`, `info`, both When using `info`, prefer using JSON strings if rows may have different schemas, e.g. different fields or nested structures: ```python -dataset = Dataset.from_list([ - {"prompt": [...], "info": '{"type": "math", "difficulty": 3}'}, - {"prompt": [...], "info": '{"type": "code", "language": "python"}'}, -]) +dataset = Dataset.from_list( + [ + {"prompt": [...], "info": '{"type": "math", "difficulty": 3}'}, + {"prompt": [...], "info": '{"type": "code", "language": "python"}'}, + ] +) ``` These are parsed into a `dict` by the environment when running rollouts. @@ -97,9 +102,11 @@ These are parsed into a `dict` by the environment when running rollouts. The examples above use `prompt` directly, providing a list of messages ready to send to the model. Alternatively, you can provide a `question` column containing a string, and the environment will wrap it in a user message: ```python -dataset = Dataset.from_list([ - {"question": "What is 2+2?", "answer": "4"}, -]) +dataset = Dataset.from_list( + [ + {"question": "What is 2+2?", "answer": "4"}, + ] +) ``` You can also pass a `system_prompt` to the environment, which prepends a system message: @@ -117,7 +124,7 @@ Together, these construct the full prompt: ```python [ {"role": "system", "content": "You are a helpful math tutor."}, - {"role": "user", "content": "What is 2+2?"} + {"role": "user", "content": "What is 2+2?"}, ] ``` @@ -144,19 +151,22 @@ For large datasets or when running multiple environment replicas, you can defer ```python def get_dataset_builder(split: str = "train", seed: int = 42) -> vf.DatasetBuilder: """Returns a builder that lazily loads the dataset.""" + def build() -> Dataset: ds = load_dataset("my-dataset", split=split) ds = ds.shuffle(seed=seed) return ds + return build + def load_environment(): dataset_builder = get_dataset_builder(split="train") eval_builder = get_dataset_builder(split="test") return vf.SingleTurnEnv( - dataset=dataset_builder, # built on first access - eval_dataset=eval_builder, # built on first access + dataset=dataset_builder, # built on first access + eval_dataset=eval_builder, # built on first access rubric=rubric, ) ``` @@ -204,14 +214,13 @@ async def check_keywords(completion, info) -> float: found = sum(1 for kw in keywords if kw.lower() in response.lower()) return found / len(keywords) + async def length_reward(completion) -> float: response = completion[-1]["content"] return 1.0 if len(response) < 500 else 0.5 -rubric = vf.Rubric( - funcs=[check_keywords, length_reward], - weights=[1.0, 0.1] -) + +rubric = vf.Rubric(funcs=[check_keywords, length_reward], weights=[1.0, 0.1]) ``` The final rollout reward is computed as the weighted sum of all reward function scores. @@ -229,6 +238,8 @@ Beyond the final score, reward functions can be used to track metrics for observ ```python async def response_length(completion) -> float: return float(len(completion[-1]["content"])) + + rubric.add_metric(response_length) # shorthand for weight=0 ``` @@ -245,12 +256,14 @@ async def similarity_score(completion, answer, state) -> float: state["similarity"] = score return score + async def similarity_threshold(state) -> float: return 1.0 if state["similarity"] > 0.8 else 0.0 + rubric = vf.Rubric( funcs=[similarity_score, similarity_threshold], - weights=[0.0, 1.0] # log similarity, but only reward threshold + weights=[0.0, 1.0], # log similarity, but only reward threshold ) ``` @@ -270,6 +283,7 @@ async def diversity_bonus(completions) -> list[float]: # Higher reward if this response is unique return [0.2 if responses.count(r) == 1 else 0.0 for r in responses] + rubric = vf.Rubric(funcs=[correct_answer, diversity_bonus]) ``` @@ -281,6 +295,7 @@ In rubric environments, reward functions can request static helper objects that rubric = vf.Rubric(funcs=[my_reward_func]) rubric.add_class_object("my_helper", some_helper_object) + async def my_reward_func(completion, my_helper) -> float: # my_helper is now available by name return await my_helper.score(completion) @@ -295,10 +310,12 @@ judge_rubric = vf.JudgeRubric( judge_model="gpt-4.1-mini", ) + async def judge_correctness(prompt, completion, answer, judge) -> float: verdict = await judge(prompt, completion, answer) return 1.0 if "yes" in verdict.lower() else 0.0 + judge_rubric.add_reward_func(judge_correctness) ``` @@ -311,10 +328,13 @@ judge_rubric = vf.JudgeRubric( judge_model="gpt-4.1-mini", judge_prompt="""Rate the writing quality of this response from 0-10. Response: {response} -Score:""" +Score:""", ) -async def quality_score(completion, judge_client, judge_model, judge_prompt, parser) -> float: + +async def quality_score( + completion, judge_client, judge_model, judge_prompt, parser +) -> float: response = parser.parse_answer(completion) filled_prompt = judge_prompt.format(response=response) result = await judge_client.chat.completions.create( @@ -374,6 +394,7 @@ class MyMonitorRubric(vf.Rubric): async def custom_metric(self, state: vf.State) -> float: return len(state["trajectory"]) + env = vf.ToolEnv(dataset=dataset, tools=tools, rubric=rubric) env.add_rubric(MyMonitorRubric()) ``` @@ -402,6 +423,7 @@ async def calculate(expression: str) -> str: except Exception as e: return f"Error: {e}" + async def lookup(term: str) -> str: """Look up a term in the knowledge base. @@ -583,6 +605,7 @@ async def answer_submitted(self, state: vf.State) -> bool: return False return "FINAL ANSWER:" in completion[-1].get("content", "") + @vf.stop(priority=-10) # expensive validation runs last async def answer_detected(self, state: vf.State) -> bool: # only runs if cheap checks didn't already stop @@ -648,7 +671,12 @@ To end a rollout from within `env_response` (e.g., when the game ends), set `sta ```python async def env_response(self, messages: vf.Messages, state: vf.State) -> vf.Messages: if check_game_over(state): - final_message = [{"role": "user", "content": "Game over! Final score: " + str(state["score"])}] + final_message = [ + { + "role": "user", + "content": "Game over! Final score: " + str(state["score"]), + } + ] state["final_env_response"] = final_message return final_message # ... normal response logic @@ -732,6 +760,7 @@ Environments that require external API keys (e.g., for judge models or external ```python import verifiers as vf + def load_environment(api_key_var: str = "OPENAI_API_KEY") -> vf.Environment: vf.ensure_keys([api_key_var]) # now safe to use os.environ[api_key_var] @@ -816,6 +845,7 @@ with open(file, "w") as f: f.write(data) # ✅ use the built-in helper from verifiers.utils.path_utils import write_temp_file + tmp_path = await asyncio.to_thread(write_temp_file, data, ".txt") ``` @@ -826,6 +856,7 @@ from concurrent.futures import ProcessPoolExecutor executor = ProcessPoolExecutor(max_workers=4) + async def heavy_reward(data): loop = asyncio.get_event_loop() return await loop.run_in_executor(executor, cpu_bound_fn, data) diff --git a/docs/v0/overview.md b/docs/v0/overview.md index 4c07f75c7c..05fb2bc8d4 100644 --- a/docs/v0/overview.md +++ b/docs/v0/overview.md @@ -84,11 +84,14 @@ Environment modules should expose a `load_environment` function which returns an # my_env.py import verifiers as vf -def load_environment(dataset_name: str = 'gsm8k') -> vf.Environment: - dataset = vf.load_example_dataset(dataset_name) # 'question' + +def load_environment(dataset_name: str = "gsm8k") -> vf.Environment: + dataset = vf.load_example_dataset(dataset_name) # 'question' + async def correct_answer(completion, answer) -> float: - completion_ans = completion[-1]['content'] + completion_ans = completion[-1]["content"] return 1.0 if completion_ans == answer else 0.0 + rubric = vf.Rubric(funcs=[correct_answer]) env = vf.SingleTurnEnv(dataset=dataset, rubric=rubric) return env diff --git a/docs/v0/reference.md b/docs/v0/reference.md index 8f38f3cc30..02c57abd62 100644 --- a/docs/v0/reference.md +++ b/docs/v0/reference.md @@ -71,6 +71,7 @@ Generation parameters passed to the inference server (e.g., `temperature`, `top_ SystemPrompt = PromptInput | SystemPromptConfig | None SystemPromptStrategy = Literal["HT", "TH", "H_OR_T", "T_OR_H", "H", "T", "REJECT"] + class SystemPromptConfig: path: str | None = None messages: list[JsonData] = [] @@ -147,9 +148,9 @@ A `dict` subclass that tracks rollout information. Accessing keys in `INPUT_FIEL ```python class RolloutInput(TypedDict): - prompt: Messages # Required - answer: str # Optional - info: Info # Optional + prompt: Messages # Required + answer: str # Optional + info: Info # Optional ``` ### RolloutOutput @@ -214,8 +215,12 @@ class TrajectoryStepTokens(TypedDict): overlong_prompt: bool is_truncated: bool routed_experts: RoutedExpertsPayload | None - multi_modal_data: NotRequired[Any] # renderers.MultiModalData sidecar (pixel_values, placeholder ranges) — set only on multimodal rollouts - prompt_attribution: NotRequired[Any] # renderers.RenderedTokens fields as a dict (per-token is_content / sampled_mask / message_indices / message_roles) — set only on RendererClient rollouts + multi_modal_data: NotRequired[ + Any + ] # renderers.MultiModalData sidecar (pixel_values, placeholder ranges) — set only on multimodal rollouts + prompt_attribution: NotRequired[ + Any + ] # renderers.RenderedTokens fields as a dict (per-token is_content / sampled_mask / message_indices / message_roles) — set only on RendererClient rollouts ``` Token-level data for training. @@ -225,8 +230,9 @@ Token-level data for training. ```python class TimeSpan(CustomBaseModel): """A timed span. duration = end - start.""" - start: float = 0.0 # Unix timestamp (seconds since epoch) - end: float = 0.0 # Unix timestamp (seconds since epoch) + + start: float = 0.0 # Unix timestamp (seconds since epoch) + end: float = 0.0 # Unix timestamp (seconds since epoch) # duration: float (computed_field) ``` @@ -235,6 +241,7 @@ class TimeSpan(CustomBaseModel): ```python class TimeSpans(CustomBaseModel): """A list of TimeSpan with aggregate duration (sum).""" + spans: list[TimeSpan] = [] # duration: float (computed_field) ``` @@ -244,12 +251,13 @@ class TimeSpans(CustomBaseModel): ```python class RolloutTiming(CustomBaseModel): """Rollout-level timing. All values in seconds.""" - start_time: float # wall-clock at rollout start - setup: TimeSpan = TimeSpan() # setup_state() span - generation: TimeSpan = TimeSpan() # full generation phase - scoring: TimeSpan = TimeSpan() # rubric.score_*() span - model: TimeSpans = TimeSpans() # all model-call spans - env: TimeSpans = TimeSpans() # all env-response spans + + start_time: float # wall-clock at rollout start + setup: TimeSpan = TimeSpan() # setup_state() span + generation: TimeSpan = TimeSpan() # full generation phase + scoring: TimeSpan = TimeSpan() # rubric.score_*() span + model: TimeSpans = TimeSpans() # all model-call spans + env: TimeSpans = TimeSpans() # all env-response spans # total, overhead: float (computed_fields) ``` @@ -300,6 +308,7 @@ class VersionInfo(TypedDict): env_version: str | None env_commit: str | None + class GenerateMetadata(TypedDict): env_id: str name: NotRequired[str] @@ -339,6 +348,7 @@ class RolloutScore(TypedDict): reward: float metrics: dict[str, float] + class RolloutScores(TypedDict): reward: list[float] metrics: dict[str, list[float]] @@ -580,7 +590,7 @@ No-agent debugger for sandbox-backed `SandboxTaskSet` instances. It creates the ```python env_group = vf.EnvGroup( envs=[env1, env2, env3], - env_names=["math", "code", "qa"] # optional + env_names=["math", "code", "qa"], # optional ) ``` @@ -685,9 +695,8 @@ def my_reward( state: State | None = None, parser: Parser | None = None, # if rubric has parser info: Info | None = None, - **kwargs -) -> float: - ... + **kwargs, +) -> float: ... ``` **Group reward function signature:** @@ -698,9 +707,8 @@ def my_group_reward( answers: list[str], states: list[State], # ... plural versions of individual args - **kwargs -) -> list[float]: - ... + **kwargs, +) -> list[float]: ... ``` #### JudgeRubric @@ -780,6 +788,7 @@ class Response(BaseModel): usage: Usage | None message: ResponseMessage + class ResponseMessage(BaseModel): content: str | None reasoning_content: str | None @@ -931,8 +940,7 @@ class PrimeCLIPlugin: def build_module_command( self, module_name: str, args: Sequence[str] | None = None - ) -> list[str]: - ... + ) -> list[str]: ... ``` `build_module_command` returns a subprocess command list for `python -m ...`. @@ -940,8 +948,7 @@ class PrimeCLIPlugin: ### get_plugin ```python -def get_plugin() -> PrimeCLIPlugin: - ... +def get_plugin() -> PrimeCLIPlugin: ... ``` Returns the plugin instance consumed by `prime`. @@ -958,9 +965,9 @@ async def my_condition(self, state: State) -> bool: """Return True to end the rollout.""" ... + @vf.stop(priority=10) # Higher priority runs first -async def early_check(self, state: State) -> bool: - ... +async def early_check(self, state: State) -> bool: ... ``` Mark a method as a stop condition. All stop conditions are checked by `is_completed()`. @@ -973,9 +980,9 @@ async def my_cleanup(self, state: State) -> None: """Called after each rollout completes.""" ... + @vf.cleanup(priority=10) -async def early_cleanup(self, state: State) -> None: - ... +async def early_cleanup(self, state: State) -> None: ... ``` Mark a method as a rollout cleanup handler. Cleanup methods should be **idempotent**—safe to call multiple times—and handle errors gracefully to ensure cleanup completes even when resources are in unexpected states. @@ -988,9 +995,9 @@ async def my_teardown(self) -> None: """Called when environment is destroyed.""" ... + @vf.teardown(priority=10) -async def early_teardown(self) -> None: - ... +async def early_teardown(self) -> None: ... ``` Mark a method as an environment teardown handler. diff --git a/docs/v1/agent.md b/docs/v1/agent.md index 9afdd88f0b..7cb42b31f0 100644 --- a/docs/v1/agent.md +++ b/docs/v1/agent.md @@ -49,7 +49,9 @@ from verifiers.v1.interception import InterceptionServer async with InterceptionServer() as server: solver = vf.make_agent(vf.AgentConfig(model="z-ai/glm-5.2"), interception=server) - judge = vf.make_agent(vf.AgentConfig(model="openai/gpt-5.4-mini"), interception=server) + judge = vf.make_agent( + vf.AgentConfig(model="openai/gpt-5.4-mini"), interception=server + ) ... ``` @@ -80,14 +82,18 @@ Agents are chained via `vf.Task`. For example, a common pattern is one agent's t class ProposedData(vf.TaskData): answer: str + class ProposedTask(vf.Task[ProposedData]): @classmethod - def from_trace(cls, proposed: vf.Trace) -> "ProposedTask": - ... # parse the proposer's contract into ProposedData + def from_trace( + cls, proposed: vf.Trace + ) -> "ProposedTask": ... # parse the proposer's contract into ProposedData @vf.reward - async def correct(self, trace: vf.Trace) -> float: - ... # compare the trace's final answer against self.data.answer + async def correct( + self, trace: vf.Trace + ) -> float: ... # compare the trace's final answer against self.data.answer + proposed = await proposer.run(vf.Task(vf.TaskData(prompt=PROPOSE))) task = ProposedTask.from_trace(proposed) diff --git a/docs/v1/harbor.md b/docs/v1/harbor.md index 85717b7e6f..1c55d29428 100644 --- a/docs/v1/harbor.md +++ b/docs/v1/harbor.md @@ -6,13 +6,16 @@ verifiers offers built-in support for Harbor via the `HarborTaskset` class. Crea import verifiers.v1 as vf from verifiers.v1.tasksets.harbor import HarborConfig, HarborTask, HarborTaskset + # Set the dataset to the same name as registered in the Harbor registry class TerminalBench2Config(HarborConfig): dataset: str = "terminal-bench/terminal-bench-2" # The data will get loaded automatically -class TerminalBench2Taskset(HarborTaskset, vf.Taskset[HarborTask, TerminalBench2Config]): +class TerminalBench2Taskset( + HarborTaskset, vf.Taskset[HarborTask, TerminalBench2Config] +): pass ``` @@ -29,19 +32,27 @@ IMAGE_TEMPLATE = "registry.example.com/openthoughts/{task}:latest" class OpenThoughtsTBLiteConfig(HarborConfig): - dataset: Literal["openthoughts/openthoughts-tblite"] = "openthoughts/openthoughts-tblite" + dataset: Literal["openthoughts/openthoughts-tblite"] = ( + "openthoughts/openthoughts-tblite" + ) # Tell verifiers to use the pre-built image ignore_dockerfile: bool = True -class OpenThoughtsTBLiteTaskset(HarborTaskset, vf.Taskset[HarborTask, OpenThoughtsTBLiteConfig]): +class OpenThoughtsTBLiteTaskset( + HarborTaskset, vf.Taskset[HarborTask, OpenThoughtsTBLiteConfig] +): def load(self) -> list[HarborTask]: # Use the public image instead to avoid building the image at runtime; the row # data is frozen, so rebuild each task around an updated copy. return [ HarborTask( task.data.model_copy( - update={"image": IMAGE_TEMPLATE.format(task=Path(task.data.task_dir).name)} + update={ + "image": IMAGE_TEMPLATE.format( + task=Path(task.data.task_dir).name + ) + } ), task.config, ) diff --git a/docs/v1/harnesses.md b/docs/v1/harnesses.md index 752f3d2735..e104e7260d 100644 --- a/docs/v1/harnesses.md +++ b/docs/v1/harnesses.md @@ -12,10 +12,12 @@ from verifiers.v1.runtimes import ProgramResult, Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace + class MyHarnessConfig(HarnessConfig): # These are the values that the users are allowed to set and change. version: str = "0.0.1" + class MyHarness(Harness[MyHarnessConfig]): # Set the system prompt of the task as the harness system message; else add it to the first user message APPENDS_SYSTEM_PROMPT = True diff --git a/docs/v1/tasksets.md b/docs/v1/tasksets.md index 76145f61bd..01252ffcc6 100644 --- a/docs/v1/tasksets.md +++ b/docs/v1/tasksets.md @@ -97,12 +97,14 @@ Keep values on the narrowest object that needs them: class AdditionTaskConfig(vf.TaskConfig): tolerance: float = 0.0 + class AdditionTask(vf.Task[AdditionData, vf.State, AdditionTaskConfig]): @vf.reward async def exact_match(self, trace: vf.Trace) -> float: error = abs(float(trace.last_reply) - self.data.answer) return float(error <= self.config.tolerance) + class AdditionConfig(vf.TasksetConfig): num_tasks: int = 100 task: AdditionTaskConfig = AdditionTaskConfig() @@ -143,6 +145,7 @@ You can create them like this (remember the bootstrapping with `uv run init MY_E ```python DATABASE = None + class SearchToolset(vf.Toolset[vf.SharedToolsetConfig]): TOOL_PREFIX = "search" @@ -151,10 +154,12 @@ class SearchToolset(vf.Toolset[vf.SharedToolsetConfig]): """Search the task corpus.""" return DATABASE.search(text) + # User-configurable knobs class SearchConfig(vf.TasksetConfig): tools: vf.SharedToolsetConfig = vf.SharedToolsetConfig() + class SearchTaskset(vf.Taskset[vf.Task, SearchConfig]): tools = (SearchToolset,) ``` @@ -169,9 +174,11 @@ If your reward is semantic, use an LLM judge. import verifiers.v1 as vf from functools import cached_property + class Task(vf.Task): answer: str + class CorrectnessJudge(vf.Judge[bool]): # The rubric for the judge prompt = """Question: {question} diff --git a/environments/alphabet_sort_v1/alphabet_sort_v1/__init__.py b/environments/alphabet_sort_v1/alphabet_sort_v1/__init__.py index 16b18499aa..fdecbbb3c7 100644 --- a/environments/alphabet_sort_v1/alphabet_sort_v1/__init__.py +++ b/environments/alphabet_sort_v1/alphabet_sort_v1/__init__.py @@ -1,3 +1,3 @@ from alphabet_sort_v1.taskset import AlphabetSortEnv, AlphabetSortTaskset -__all__ = ["AlphabetSortTaskset", "AlphabetSortEnv"] +__all__ = ["AlphabetSortEnv", "AlphabetSortTaskset"] diff --git a/environments/alphabet_sort_v1/alphabet_sort_v1/taskset.py b/environments/alphabet_sort_v1/alphabet_sort_v1/taskset.py index 42adf2e75c..0ed4b52930 100644 --- a/environments/alphabet_sort_v1/alphabet_sort_v1/taskset.py +++ b/environments/alphabet_sort_v1/alphabet_sort_v1/taskset.py @@ -20,6 +20,7 @@ import random import re from collections.abc import Iterator +from itertools import pairwise from typing import Literal from datasets import load_dataset @@ -91,7 +92,7 @@ async def alphabet_sort(self, trace: vf.Trace) -> float: elif len(attempts) == 1: scores.append(attempts[0]) else: - improved = all(b > a for a, b in zip(attempts, attempts[1:])) + improved = all(b > a for a, b in pairwise(attempts)) scores.append(attempts[-1] if improved else 0.0) avg = sum(scores) / num_turns if num_turns else 0.0 return avg if self.config.power_per_turn else avg**self.config.similarity_power @@ -136,7 +137,7 @@ def load(self) -> Iterator[AlphabetSortTask]: by_first = rng.choice([True, False]) label = "FIRST" if by_first else "LAST" - def sort_key(s: str) -> str: + def sort_key(s: str, by_first: bool = by_first) -> str: # split at the first capital after index 0 -> first- vs last-name part cut = next((i for i in range(1, len(s)) if s[i].isupper()), len(s)) return s[:cut] if by_first else s[cut:] diff --git a/environments/code_golf_v1/code_golf_v1/__init__.py b/environments/code_golf_v1/code_golf_v1/__init__.py index c8db9f0c3c..09c29c86c5 100644 --- a/environments/code_golf_v1/code_golf_v1/__init__.py +++ b/environments/code_golf_v1/code_golf_v1/__init__.py @@ -1,3 +1,3 @@ from code_golf_v1.taskset import CodeGolfEnv, CodeGolfTaskset -__all__ = ["CodeGolfTaskset", "CodeGolfEnv"] +__all__ = ["CodeGolfEnv", "CodeGolfTaskset"] diff --git a/environments/code_golf_v1/code_golf_v1/taskset.py b/environments/code_golf_v1/code_golf_v1/taskset.py index 11a4907841..32ac8d5a89 100644 --- a/environments/code_golf_v1/code_golf_v1/taskset.py +++ b/environments/code_golf_v1/code_golf_v1/taskset.py @@ -22,6 +22,7 @@ import asyncio import re import time +from typing import ClassVar from pydantic import Field @@ -99,7 +100,7 @@ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: class CodeGolfTaskset(vf.Taskset[CodeGolfTask, vf.TasksetConfig]): # (name, description, expected stdout) - SPECS = [ + SPECS: ClassVar = [ ("simple-sum", "the sum of the integers 1 to 100", "5050"), ( "fibonacci", diff --git a/environments/color_codeword_v1/color_codeword_v1/__init__.py b/environments/color_codeword_v1/color_codeword_v1/__init__.py index 511c0d8453..c1b1334655 100644 --- a/environments/color_codeword_v1/color_codeword_v1/__init__.py +++ b/environments/color_codeword_v1/color_codeword_v1/__init__.py @@ -1,3 +1,3 @@ from color_codeword_v1.taskset import ColorCodewordEnv, ColorCodewordTaskset -__all__ = ["ColorCodewordTaskset", "ColorCodewordEnv"] +__all__ = ["ColorCodewordEnv", "ColorCodewordTaskset"] diff --git a/environments/deepwiki_v1/deepwiki_v1/taskset.py b/environments/deepwiki_v1/deepwiki_v1/taskset.py index 57edaf1696..7a9e79dabf 100644 --- a/environments/deepwiki_v1/deepwiki_v1/taskset.py +++ b/environments/deepwiki_v1/deepwiki_v1/taskset.py @@ -7,7 +7,6 @@ """ import verifiers.v1 as vf - from deepwiki_v1.servers.deepwiki import DEEPWIKI_URL, DeepWikiToolset # (repository, expected primary language) pairs chosen for unambiguous answers. diff --git a/environments/glossary_v1/glossary_v1/taskset.py b/environments/glossary_v1/glossary_v1/taskset.py index 1c6d88b239..c1e26f6de6 100644 --- a/environments/glossary_v1/glossary_v1/taskset.py +++ b/environments/glossary_v1/glossary_v1/taskset.py @@ -6,7 +6,6 @@ """ import verifiers.v1 as vf - from glossary_v1.servers.facts import FACTS, GlossaryToolset diff --git a/environments/gsm8k_v1/gsm8k_v1/verify.py b/environments/gsm8k_v1/gsm8k_v1/verify.py index a88473cc86..b02a09b437 100644 --- a/environments/gsm8k_v1/gsm8k_v1/verify.py +++ b/environments/gsm8k_v1/gsm8k_v1/verify.py @@ -26,6 +26,6 @@ if verify(parse("\\boxed{" + gold + "}"), parse("\\boxed{" + prediction + "}")) else 0.0 ) -except Exception: +except Exception: # noqa: BLE001 - malformed answers can fail anywhere in math-verify score = 0.0 print(score) diff --git a/environments/kuhn_poker_v1/kuhn_poker_v1/__init__.py b/environments/kuhn_poker_v1/kuhn_poker_v1/__init__.py index a03ad8bb31..ed490f42b5 100644 --- a/environments/kuhn_poker_v1/kuhn_poker_v1/__init__.py +++ b/environments/kuhn_poker_v1/kuhn_poker_v1/__init__.py @@ -1,3 +1,3 @@ from kuhn_poker_v1.taskset import KuhnPokerEnv, KuhnPokerTaskset -__all__ = ["KuhnPokerTaskset", "KuhnPokerEnv"] +__all__ = ["KuhnPokerEnv", "KuhnPokerTaskset"] diff --git a/environments/kuhn_poker_v1/kuhn_poker_v1/taskset.py b/environments/kuhn_poker_v1/kuhn_poker_v1/taskset.py index afdec4f789..3703832227 100644 --- a/environments/kuhn_poker_v1/kuhn_poker_v1/taskset.py +++ b/environments/kuhn_poker_v1/kuhn_poker_v1/taskset.py @@ -16,8 +16,8 @@ import random import re +from collections.abc import Iterator from itertools import count -from typing import Iterator from pydantic import Field diff --git a/environments/openenv_wordle_v1/openenv_wordle_v1/__init__.py b/environments/openenv_wordle_v1/openenv_wordle_v1/__init__.py index 14e00b8721..960de8c679 100644 --- a/environments/openenv_wordle_v1/openenv_wordle_v1/__init__.py +++ b/environments/openenv_wordle_v1/openenv_wordle_v1/__init__.py @@ -1,3 +1,3 @@ from openenv_wordle_v1.taskset import OpenEnvEnv, OpenEnvWordleTaskset -__all__ = ["OpenEnvWordleTaskset", "OpenEnvEnv"] +__all__ = ["OpenEnvEnv", "OpenEnvWordleTaskset"] diff --git a/environments/openenv_wordle_v1/openenv_wordle_v1/taskset.py b/environments/openenv_wordle_v1/openenv_wordle_v1/taskset.py index 75df1b47d7..e4aed6c1ef 100644 --- a/environments/openenv_wordle_v1/openenv_wordle_v1/taskset.py +++ b/environments/openenv_wordle_v1/openenv_wordle_v1/taskset.py @@ -10,13 +10,13 @@ OpenEnvTaskset, ) -__all__ = ["OpenEnvWordleConfig", "OpenEnvWordleTaskset", "OpenEnvEnv"] +__all__ = ["OpenEnvEnv", "OpenEnvWordleConfig", "OpenEnvWordleTaskset"] class OpenEnvWordleConfig(OpenEnvConfig): env: Literal["openenv/wordle"] = "openenv/wordle" - def model_post_init(self, __context: Any) -> None: + def model_post_init(self, __context: Any, /) -> None: # Wordle uses a non-default ASGI app in its Space repository. self.provider_kwargs.setdefault("app", "textarena_env.server.app:app") diff --git a/environments/proposer_solver_v1/proposer_solver_v1/__init__.py b/environments/proposer_solver_v1/proposer_solver_v1/__init__.py index 2a81cfece6..4ac12449a3 100644 --- a/environments/proposer_solver_v1/proposer_solver_v1/__init__.py +++ b/environments/proposer_solver_v1/proposer_solver_v1/__init__.py @@ -4,4 +4,4 @@ ProposerSolverTaskset, ) -__all__ = ["ProposerSolverTaskset", "ProposerSolverEnv", "ProposerSolverEnvConfig"] +__all__ = ["ProposerSolverEnv", "ProposerSolverEnvConfig", "ProposerSolverTaskset"] diff --git a/environments/proposer_solver_v1/proposer_solver_v1/taskset.py b/environments/proposer_solver_v1/proposer_solver_v1/taskset.py index b023f8dbb1..9b424f82ca 100644 --- a/environments/proposer_solver_v1/proposer_solver_v1/taskset.py +++ b/environments/proposer_solver_v1/proposer_solver_v1/taskset.py @@ -25,6 +25,7 @@ import asyncio import json import re +from typing import ClassVar from pydantic import Field @@ -102,7 +103,7 @@ def from_trace(cls, proposer: vf.Trace) -> "SolveTask": ) answer = proposed["answer"] if not isinstance(answer, int) or isinstance(answer, bool): - raise ValueError( + raise TypeError( f"proposer's contract answer is not a JSON integer: {answer!r}" ) return cls( @@ -168,7 +169,7 @@ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: class ProposerSolverTaskset(vf.Taskset[ProposeTask, vf.TasksetConfig]): - TOPICS = [ + TOPICS: ClassVar = [ "rates and mixtures", "number theory", "combinatorics", diff --git a/environments/scratchpad_v1/scratchpad_v1/taskset.py b/environments/scratchpad_v1/scratchpad_v1/taskset.py index 65ec7cf711..c56be8931c 100644 --- a/environments/scratchpad_v1/scratchpad_v1/taskset.py +++ b/environments/scratchpad_v1/scratchpad_v1/taskset.py @@ -9,7 +9,6 @@ """ import verifiers.v1 as vf - from scratchpad_v1.servers.scratchpad import ( ScratchpadState, ScratchpadToolset, diff --git a/environments/wiki_search_v1/wiki_search_v1/taskset.py b/environments/wiki_search_v1/wiki_search_v1/taskset.py index 37a205b56a..39e894cb67 100644 --- a/environments/wiki_search_v1/wiki_search_v1/taskset.py +++ b/environments/wiki_search_v1/wiki_search_v1/taskset.py @@ -6,8 +6,9 @@ whose prompt also rejects incoherent answers. """ -import verifiers.v1 as vf +from pydantic import Field +import verifiers.v1 as vf from wiki_search_v1.servers.wiki import WikiSearchToolset SYSTEM = ( @@ -45,9 +46,11 @@ class WikiSearchTaskConfig(vf.TaskConfig): # Users can replace or reconfigure this judge through --taskset.task.judges. - judges: vf.Judges = [ - vf.ReferenceJudgeConfig(prompt=JUDGE_PROMPT, question_field="question") - ] + judges: vf.Judges = Field( + default_factory=lambda: [ + vf.ReferenceJudgeConfig(prompt=JUDGE_PROMPT, question_field="question") + ] + ) class TriviaTaskData(vf.TaskData): diff --git a/environments/wordle_v1/wordle_v1/__init__.py b/environments/wordle_v1/wordle_v1/__init__.py index 119d08346a..cef4ae69b8 100644 --- a/environments/wordle_v1/wordle_v1/__init__.py +++ b/environments/wordle_v1/wordle_v1/__init__.py @@ -1,4 +1,4 @@ from verifiers.v1.tasksets.textarena import TextArenaEnv from wordle_v1.taskset import WordleTaskset -__all__ = ["WordleTaskset", "TextArenaEnv"] +__all__ = ["TextArenaEnv", "WordleTaskset"] diff --git a/tests/conftest.py b/tests/conftest.py index 166c1f87a9..11a3b25db1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Pytest configuration and fixtures for verifiers tests.""" import logging diff --git a/tests/test_browser_env.py b/tests/test_browser_env.py index 9c11cd6c1e..5d146e9e35 100644 --- a/tests/test_browser_env.py +++ b/tests/test_browser_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for BrowserEnv integration (browser_env, dom_mode, cua_mode). These tests verify the BrowserEnv class and its mode implementations diff --git a/tests/test_cli_agent_env.py b/tests/test_cli_agent_env.py index 041ec3505a..9628c54d52 100644 --- a/tests/test_cli_agent_env.py +++ b/tests/test_cli_agent_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for CliAgentEnv and HarborEnv.""" import asyncio diff --git a/tests/test_client_auth_errors.py b/tests/test_client_auth_errors.py index 83141a26fc..cb5f38f177 100644 --- a/tests/test_client_auth_errors.py +++ b/tests/test_client_auth_errors.py @@ -1,3 +1,5 @@ +# ruff: noqa + import httpx import pytest from anthropic import AuthenticationError as AnthropicAuthenticationError diff --git a/tests/test_composable_env.py b/tests/test_composable_env.py index 94ade63e13..2752e633d7 100644 --- a/tests/test_composable_env.py +++ b/tests/test_composable_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for the composable architecture: Task, TaskSet, SandboxTaskSet, SandboxSpec.""" import importlib diff --git a/tests/test_context_token_metrics.py b/tests/test_context_token_metrics.py index 37bb8e69c9..4efc2c2b4b 100644 --- a/tests/test_context_token_metrics.py +++ b/tests/test_context_token_metrics.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for per-turn context token metrics. Tests the trajectory-based context token computation diff --git a/tests/test_decorator_ranks.py b/tests/test_decorator_ranks.py index bc2795def2..829727492d 100644 --- a/tests/test_decorator_ranks.py +++ b/tests/test_decorator_ranks.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for decorator priority ordering functionality.""" import pytest diff --git a/tests/test_env_server.py b/tests/test_env_server.py index 58348afc70..0a3eb0dab4 100644 --- a/tests/test_env_server.py +++ b/tests/test_env_server.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for the env server and client. Covers: diff --git a/tests/test_environment_extra.py b/tests/test_environment_extra.py index 3854a70290..12a43699e1 100644 --- a/tests/test_environment_extra.py +++ b/tests/test_environment_extra.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Additional tests for verifiers.envs.environment.Environment. Covers: diff --git a/tests/test_gepa_cli.py b/tests/test_gepa_cli.py index 017f8605d9..1b51276939 100644 --- a/tests/test_gepa_cli.py +++ b/tests/test_gepa_cli.py @@ -1,3 +1,5 @@ +# ruff: noqa + import argparse from pathlib import Path diff --git a/tests/test_imports.py b/tests/test_imports.py index 44f7283285..eeb49d0aab 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -1,3 +1,5 @@ +# ruff: noqa + import verifiers diff --git a/tests/test_nemorl_client.py b/tests/test_nemorl_client.py index 89a721ae9e..3fb6958ff8 100644 --- a/tests/test_nemorl_client.py +++ b/tests/test_nemorl_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + from unittest.mock import AsyncMock, patch import pytest diff --git a/tests/test_openai_chat_completions_token_client.py b/tests/test_openai_chat_completions_token_client.py index 923ff118e0..16c2e32de9 100644 --- a/tests/test_openai_chat_completions_token_client.py +++ b/tests/test_openai_chat_completions_token_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import Any, cast import httpx diff --git a/tests/test_opencode_rlm_env.py b/tests/test_opencode_rlm_env.py index 8951bd9619..151574bb06 100644 --- a/tests/test_opencode_rlm_env.py +++ b/tests/test_opencode_rlm_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for the OpenCodeRLMEnv class.""" import asyncio diff --git a/tests/test_openenv_client.py b/tests/test_openenv_client.py index 7672053094..b89f4d5b6c 100644 --- a/tests/test_openenv_client.py +++ b/tests/test_openenv_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import Any import verifiers as vf diff --git a/tests/test_renderer_client.py b/tests/test_renderer_client.py index 566b46afd0..c07d8e5c0b 100644 --- a/tests/test_renderer_client.py +++ b/tests/test_renderer_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio from functools import lru_cache from unittest.mock import patch diff --git a/tests/test_renderer_e2e.py b/tests/test_renderer_e2e.py index 681b6aa4ef..aae57d630e 100644 --- a/tests/test_renderer_e2e.py +++ b/tests/test_renderer_e2e.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tier-1 end-to-end tests for the renderer → RendererClient → Environment path. Exercises the full TITO wire protocol with a scripted stand-in for vLLM: diff --git a/tests/test_rlm_composable_env.py b/tests/test_rlm_composable_env.py index 52b232a541..c248ead6a3 100644 --- a/tests/test_rlm_composable_env.py +++ b/tests/test_rlm_composable_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for RLM harness integration with ComposableEnv. Validates that rlm_harness() produces a Harness with the correct metrics diff --git a/tests/test_rubric.py b/tests/test_rubric.py index 8b2293364a..7ef4e32b13 100644 --- a/tests/test_rubric.py +++ b/tests/test_rubric.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for the Rubric class.""" from typing import cast diff --git a/tests/test_sandbox_mixin.py b/tests/test_sandbox_mixin.py index 79c3ddaca3..4e671745bb 100644 --- a/tests/test_sandbox_mixin.py +++ b/tests/test_sandbox_mixin.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/test_save_utils.py b/tests/test_save_utils.py index 4c4fdf3804..6d57099eba 100644 --- a/tests/test_save_utils.py +++ b/tests/test_save_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for verifiers.utils.save_utils serialization behavior. Covers: diff --git a/tests/test_tool_utils.py b/tests/test_tool_utils.py index c369cdda0d..7ddc357dd7 100644 --- a/tests/test_tool_utils.py +++ b/tests/test_tool_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for the tool_utils module.""" from typing import Annotated, Optional diff --git a/tests/v1/conftest.py b/tests/v1/conftest.py index c0ca3c9dc9..79aee17565 100644 --- a/tests/v1/conftest.py +++ b/tests/v1/conftest.py @@ -37,9 +37,9 @@ import pytest import verifiers.v1 as vf +from verifiers.v1.cli.eval.runner import run_eval from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.loaders import load_environment -from verifiers.v1.cli.eval.runner import run_eval from verifiers.v1.trace import Trace # Fixture tasksets/envs (echo-v1, echo-agentic-v1, echo-v0, echo-multi-v0) live in diff --git a/tests/v1/fixtures/duet_v1.py b/tests/v1/fixtures/duet_v1.py index c47448aeb5..3cd2ccf450 100644 --- a/tests/v1/fixtures/duet_v1.py +++ b/tests/v1/fixtures/duet_v1.py @@ -9,10 +9,10 @@ import asyncio -import verifiers.v1 as vf - from echo_v1 import EchoTaskset, lenient_match +import verifiers.v1 as vf + class DuetEnvConfig(vf.EnvConfig): # Both seats pin the lean `null` chat loop. @@ -42,4 +42,4 @@ class DuetTaskset(EchoTaskset): pass -__all__ = ["DuetTaskset", "DuetEnv"] +__all__ = ["DuetEnv", "DuetTaskset"] diff --git a/tests/v1/fixtures/echo_acp_resume_v1.py b/tests/v1/fixtures/echo_acp_resume_v1.py index ef93726ac8..801896ba52 100644 --- a/tests/v1/fixtures/echo_acp_resume_v1.py +++ b/tests/v1/fixtures/echo_acp_resume_v1.py @@ -93,7 +93,7 @@ def load(self) -> list[ACPResumeTask]: ] -__all__ = ["ACPResumeTaskset", "ACPResumeEnv"] +__all__ = ["ACPResumeEnv", "ACPResumeTaskset"] if __name__ == "__main__": diff --git a/tests/v1/fixtures/echo_user_sim_v1.py b/tests/v1/fixtures/echo_user_sim_v1.py index 89d9f0cfff..f57d0d7dd2 100644 --- a/tests/v1/fixtures/echo_user_sim_v1.py +++ b/tests/v1/fixtures/echo_user_sim_v1.py @@ -62,4 +62,4 @@ def load(self) -> list[EchoUserSimTask]: ] -__all__ = ["EchoUserSimTaskset", "EchoUserSimEnv"] +__all__ = ["EchoUserSimEnv", "EchoUserSimTaskset"] diff --git a/tests/v1/fixtures/echo_v1.py b/tests/v1/fixtures/echo_v1.py index a0400405c1..17d27ff422 100644 --- a/tests/v1/fixtures/echo_v1.py +++ b/tests/v1/fixtures/echo_v1.py @@ -6,6 +6,8 @@ by id `echo-v1` via pytest's `pythonpath`). """ +from pydantic import Field + import verifiers.v1 as vf SYSTEM = "Repeat the user's message back to them exactly, with no extra words." @@ -39,7 +41,9 @@ async def echoed(self, trace: vf.Trace) -> float: class EchoConfig(vf.TasksetConfig): - phrases: list[str] = ["hello world", "ping", "verifiers"] + phrases: list[str] = Field( + default_factory=lambda: ["hello world", "ping", "verifiers"] + ) class EchoTaskset(vf.Taskset[EchoTask, EchoConfig]): diff --git a/tests/v1/test_envs.py b/tests/v1/test_envs.py index 93bbd3ea84..dee491a2d7 100644 --- a/tests/v1/test_envs.py +++ b/tests/v1/test_envs.py @@ -72,7 +72,9 @@ def test_eval(taskset: str): "--sampling.max-tokens", "512", "--rich", "false", ] # fmt: skip try: - proc = subprocess.run(cmd, capture_output=True, text=True, timeout=EVAL_TIMEOUT) + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=EVAL_TIMEOUT, check=False + ) except subprocess.TimeoutExpired: pytest.fail(f"Timed out after {EVAL_TIMEOUT}s evaluating {taskset}") assert proc.returncode == 0, ( diff --git a/tests/v1/test_judges.py b/tests/v1/test_judges.py index cf5e2f0b6f..4fafa2e0b9 100644 --- a/tests/v1/test_judges.py +++ b/tests/v1/test_judges.py @@ -6,6 +6,7 @@ import re import pytest +from pydantic import Field import verifiers.v1 as vf from verifiers.v1.graph import MessageNode @@ -72,7 +73,9 @@ async def fake_complete( "reason": "cites Paris" if "Paris" in text else "no Paris", "verdict": "yes" if "Paris" in text else "no", } - for name, text in re.findall(r"^- ([^:]+): (.+)$", messages, re.M) + for name, text in re.findall( + r"^- ([^:]+): (.+)$", messages, re.MULTILINE + ) ] response = JudgeResponse( text=json.dumps({"verdicts": verdicts}), @@ -167,7 +170,12 @@ def test_judges_reject_shared_reward_keys(): # class-level DEFAULTS are held to the same rule (they bypass the before-hook) class TwoDefaults(vf.TaskConfig): - judges: vf.Judges = [vf.ReferenceJudgeConfig(), vf.ReferenceJudgeConfig()] + judges: vf.Judges = Field( + default_factory=lambda: [ + vf.ReferenceJudgeConfig(), + vf.ReferenceJudgeConfig(), + ] + ) with pytest.raises(ValueError, match="share a reward key"): TwoDefaults() @@ -227,7 +235,8 @@ async def test_reference_score(fake_judge_model): async def test_reference_score_messages_prompt(fake_judge_model): # A Messages-form prompt still reaches the judge as text (via TaskData.prompt_text). - from verifiers.v1.types import TextContentPart, UserMessage as UM + from verifiers.v1.types import TextContentPart + from verifiers.v1.types import UserMessage as UM task = QAData( idx=0, @@ -400,7 +409,7 @@ async def test_reference_empty_response_short_circuits(fake_judge_model): async def test_reference_list_answer(fake_judge_model): # A list-valued answer field is judged as multiple acceptable answers, one per line. class MultiTask(vf.TaskData): - aliases: list[str] = [] + aliases: list[str] = Field(default_factory=list) task = MultiTask(idx=0, prompt="q?", aliases=["Paris", "Lutetia"]) trace = make_trace() @@ -569,11 +578,11 @@ async def off_menu(self, messages, *, trace=None, schema=None, parse=None, **s): def test_rubric_choices_validation(tmp_path): with pytest.raises(ValueError, match="at least two"): - rubric_judge( + _ = rubric_judge( tmp_path, body='[[criteria]]\nname = "x"\ntext = "t"\nchoices = ["only"]\n' ).criteria with pytest.raises(ValueError, match="duplicate options"): - rubric_judge( + _ = rubric_judge( tmp_path, body='[[criteria]]\nname = "x"\ntext = "t"\nchoices = ["a", "a"]\n', ).criteria diff --git a/tests/v1/test_scoring.py b/tests/v1/test_scoring.py index 57bc447fdd..d687212df6 100644 --- a/tests/v1/test_scoring.py +++ b/tests/v1/test_scoring.py @@ -10,13 +10,11 @@ def test_compare_stdout_results_keeps_numeric_tolerance() -> None: def test_parse_pytest_outcomes_strips_xfail_xpass_reasons() -> None: - output = "\n".join( - [ - "XFAIL tests/test_mod.py::test_xfail - known bug - still tracked", - "XPASS tests/test_mod.py::test_xpass - always xfail - unexpectedly passed", - "FAILED tests/test_mod.py::test_param[a - b] - assert left - right", - "PASSED tests/test_mod.py::test_ok", - ] + output = ( + "XFAIL tests/test_mod.py::test_xfail - known bug - still tracked\n" + "XPASS tests/test_mod.py::test_xpass - always xfail - unexpectedly passed\n" + "FAILED tests/test_mod.py::test_param[a - b] - assert left - right\n" + "PASSED tests/test_mod.py::test_ok" ) assert vf.parse_pytest_outcomes(output) == { diff --git a/verifiers/__init__.py b/verifiers/__init__.py index 384f6d57d1..57f4f0d53e 100644 --- a/verifiers/__init__.py +++ b/verifiers/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from importlib.metadata import PackageNotFoundError from importlib.metadata import version as _version diff --git a/verifiers/cli/commands/build.py b/verifiers/cli/commands/build.py index 0f8fb817ec..5f9e3fa8f6 100644 --- a/verifiers/cli/commands/build.py +++ b/verifiers/cli/commands/build.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Environment build command module for external hosts.""" from verifiers.scripts.build import main diff --git a/verifiers/cli/commands/eval.py b/verifiers/cli/commands/eval.py index 07522a4b76..a4a8368371 100644 --- a/verifiers/cli/commands/eval.py +++ b/verifiers/cli/commands/eval.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Evaluation command module for external hosts.""" from verifiers.scripts.eval import ( diff --git a/verifiers/cli/commands/gepa.py b/verifiers/cli/commands/gepa.py index 4179c1eee5..5203074d8f 100644 --- a/verifiers/cli/commands/gepa.py +++ b/verifiers/cli/commands/gepa.py @@ -1,3 +1,5 @@ +# ruff: noqa + """GEPA command module for external hosts.""" from verifiers.scripts.gepa import main diff --git a/verifiers/cli/commands/init.py b/verifiers/cli/commands/init.py index 78a98ec6a8..e6002ac375 100644 --- a/verifiers/cli/commands/init.py +++ b/verifiers/cli/commands/init.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Environment initialization command module for external hosts.""" from verifiers.scripts.init import main diff --git a/verifiers/cli/commands/install.py b/verifiers/cli/commands/install.py index 5889285a64..c30418fff0 100644 --- a/verifiers/cli/commands/install.py +++ b/verifiers/cli/commands/install.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Environment install command module for external hosts.""" from verifiers.scripts.install import main diff --git a/verifiers/cli/plugins/prime.py b/verifiers/cli/plugins/prime.py index 764fccd46e..3695d4b3a1 100644 --- a/verifiers/cli/plugins/prime.py +++ b/verifiers/cli/plugins/prime.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Prime-hosted command plugin contract.""" from dataclasses import dataclass diff --git a/verifiers/clients/__init__.py b/verifiers/clients/__init__.py index 7abbdf8b9e..83cbbad484 100644 --- a/verifiers/clients/__init__.py +++ b/verifiers/clients/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.clients.anthropic_messages_client import AnthropicMessagesClient from verifiers.clients.client import Client from verifiers.clients.nemorl_chat_completions_client import ( diff --git a/verifiers/clients/anthropic_messages_client.py b/verifiers/clients/anthropic_messages_client.py index dc803b3231..aed1c3c368 100644 --- a/verifiers/clients/anthropic_messages_client.py +++ b/verifiers/clients/anthropic_messages_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import functools import json import time diff --git a/verifiers/clients/client.py b/verifiers/clients/client.py index 18d8570bd9..9522002e00 100644 --- a/verifiers/clients/client.py +++ b/verifiers/clients/client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Generic, TypeVar diff --git a/verifiers/clients/openai_chat_completions_client.py b/verifiers/clients/openai_chat_completions_client.py index e2d00bdf5b..8f33277764 100644 --- a/verifiers/clients/openai_chat_completions_client.py +++ b/verifiers/clients/openai_chat_completions_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import functools from collections.abc import Iterable, Mapping from typing import Any, TypeAlias, cast diff --git a/verifiers/clients/openai_chat_completions_token_client.py b/verifiers/clients/openai_chat_completions_token_client.py index 2d8cd701cc..5068986f23 100644 --- a/verifiers/clients/openai_chat_completions_token_client.py +++ b/verifiers/clients/openai_chat_completions_token_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + from collections.abc import Mapping from typing import Any, Optional, cast diff --git a/verifiers/clients/openai_completions_client.py b/verifiers/clients/openai_completions_client.py index 137e7a49aa..2992d3e311 100644 --- a/verifiers/clients/openai_completions_client.py +++ b/verifiers/clients/openai_completions_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + from openai import ( AsyncOpenAI, ) diff --git a/verifiers/clients/openai_responses_client.py b/verifiers/clients/openai_responses_client.py index e86f8e4409..614dd8b60b 100644 --- a/verifiers/clients/openai_responses_client.py +++ b/verifiers/clients/openai_responses_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import time from collections.abc import Iterable, Mapping from typing import Any, TypeAlias diff --git a/verifiers/clients/renderer_client.py b/verifiers/clients/renderer_client.py index d7a58d0e87..bfd916f59c 100644 --- a/verifiers/clients/renderer_client.py +++ b/verifiers/clients/renderer_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Renderer-based client. All tokenization happens client-side via a Renderer from the renderers package. diff --git a/verifiers/decorators.py b/verifiers/decorators.py index 9b0da8edc3..e7ae789bbd 100644 --- a/verifiers/decorators.py +++ b/verifiers/decorators.py @@ -1,3 +1,5 @@ +# ruff: noqa + import inspect from typing import Any, Callable, Literal, TypeVar, overload diff --git a/verifiers/envs/env_group.py b/verifiers/envs/env_group.py index 77d96d69a1..5be67ea22c 100644 --- a/verifiers/envs/env_group.py +++ b/verifiers/envs/env_group.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json from collections.abc import Mapping from typing import TYPE_CHECKING, Any, cast, final diff --git a/verifiers/envs/environment.py b/verifiers/envs/environment.py index d2c462e7a4..80e0dd4987 100644 --- a/verifiers/envs/environment.py +++ b/verifiers/envs/environment.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import atexit import json diff --git a/verifiers/envs/experimental/__init__.py b/verifiers/envs/experimental/__init__.py index ca19f7fb19..b3dc37f4be 100644 --- a/verifiers/envs/experimental/__init__.py +++ b/verifiers/envs/experimental/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.envs.experimental.sandbox_mixin import SandboxMixin __all__ = [ diff --git a/verifiers/envs/experimental/cli_agent_env.py b/verifiers/envs/experimental/cli_agent_env.py index 7d0e017b4c..bd8092c8fa 100644 --- a/verifiers/envs/experimental/cli_agent_env.py +++ b/verifiers/envs/experimental/cli_agent_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import logging import os diff --git a/verifiers/envs/experimental/composable/README.md b/verifiers/envs/experimental/composable/README.md index c6dd5e5782..3528e40944 100644 --- a/verifiers/envs/experimental/composable/README.md +++ b/verifiers/envs/experimental/composable/README.md @@ -30,13 +30,13 @@ from verifiers.envs.experimental.composable.harnesses.opencode import opencode_h from verifiers.envs.experimental.composable import ComposableEnv # Create a taskset -taskset = R2EGymTaskSet() # 4578 SWE instances +taskset = R2EGymTaskSet() # 4578 SWE instances # Explore instances -task = taskset[0] # one Task -task.prompt # the problem statement -task.sandbox_spec # SandboxSpec(image=..., cpu=4, ...) -task.sandbox_spec.image # per-instance docker image +task = taskset[0] # one Task +task.prompt # the problem statement +task.sandbox_spec # SandboxSpec(image=..., cpu=4, ...) +task.sandbox_spec.image # per-instance docker image # Slice and filter small = taskset.take(100) @@ -46,9 +46,9 @@ filtered = taskset.filter(lambda ex: ...) results = await taskset.validate( concurrency=50, out_path="outputs/validate.jsonl", - max_retries=2, # retry on vf.InfraError + max_retries=2, # retry on vf.InfraError sandbox_client_max_workers=100, # optional sandbox client worker cap override - resume=True, # skip indices already in out_path + resume=True, # skip indices already in out_path ) # Run with an agent @@ -135,7 +135,9 @@ class MySWERubric(vf.Rubric): sandbox_client = state["sandbox_client"] sandbox_id = state["sandbox_id"] # Run tests in the sandbox - test_output = await self.taskset._run_tests(sandbox_client, sandbox_id, state, ...) + test_output = await self.taskset._run_tests( + sandbox_client, sandbox_id, state, ... + ) return float(self.taskset._calculate_reward(test_output, info)) @vf.cleanup diff --git a/verifiers/envs/experimental/composable/__init__.py b/verifiers/envs/experimental/composable/__init__.py index ecfcd1fd75..fd2a5a50e9 100644 --- a/verifiers/envs/experimental/composable/__init__.py +++ b/verifiers/envs/experimental/composable/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.envs.experimental.composable.task import ( SandboxSpec, Task, diff --git a/verifiers/envs/experimental/composable/_filter.py b/verifiers/envs/experimental/composable/_filter.py index 2ad1a9bb17..ed7cf5ce50 100644 --- a/verifiers/envs/experimental/composable/_filter.py +++ b/verifiers/envs/experimental/composable/_filter.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Helper for the base ``TaskSet.filter_fn`` kwarg. ``filter_fn`` accepts a Python expression string (typically a lambda) that diff --git a/verifiers/envs/experimental/composable/composable_env.py b/verifiers/envs/experimental/composable/composable_env.py index 8d3282dbd2..416b0ac9f2 100644 --- a/verifiers/envs/experimental/composable/composable_env.py +++ b/verifiers/envs/experimental/composable/composable_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """ComposableEnv — a CliAgentEnv that delegates to a TaskSet + Harness. Subclasses ``CliAgentEnv`` and overrides its hooks to delegate to the diff --git a/verifiers/envs/experimental/composable/harness.py b/verifiers/envs/experimental/composable/harness.py index 254fbfafe9..80edc15673 100644 --- a/verifiers/envs/experimental/composable/harness.py +++ b/verifiers/envs/experimental/composable/harness.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Harness — agent-side configuration for ComposableEnv. A Harness declares how to install and run an agent binary, and where it diff --git a/verifiers/envs/experimental/composable/harnesses/__init__.py b/verifiers/envs/experimental/composable/harnesses/__init__.py index 6c0e1deb91..be88967de8 100644 --- a/verifiers/envs/experimental/composable/harnesses/__init__.py +++ b/verifiers/envs/experimental/composable/harnesses/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.envs.experimental.composable.harnesses.rlm import ( DEFAULT_RLM_EXEC_TIMEOUT, DEFAULT_RLM_MAX_TURNS, diff --git a/verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py b/verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py index eca6710b6d..69b3df200a 100644 --- a/verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py +++ b/verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py @@ -1,3 +1,5 @@ +# ruff: noqa + """mini-SWE-agent harness configuration.""" from pathlib import PurePosixPath diff --git a/verifiers/envs/experimental/composable/harnesses/rlm.py b/verifiers/envs/experimental/composable/harnesses/rlm.py index c2a1528f3d..89bdeb0898 100644 --- a/verifiers/envs/experimental/composable/harnesses/rlm.py +++ b/verifiers/envs/experimental/composable/harnesses/rlm.py @@ -1,3 +1,5 @@ +# ruff: noqa + """RLM agent harness: install script, run command, and harness factory.""" import hashlib diff --git a/verifiers/envs/experimental/composable/sandbox_debug_env.py b/verifiers/envs/experimental/composable/sandbox_debug_env.py index 450786597b..5c2d70262b 100644 --- a/verifiers/envs/experimental/composable/sandbox_debug_env.py +++ b/verifiers/envs/experimental/composable/sandbox_debug_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """No-agent debugger for sandbox-backed TaskSet instances.""" import shlex diff --git a/verifiers/envs/experimental/composable/swe_debug_env.py b/verifiers/envs/experimental/composable/swe_debug_env.py index e68d5e5926..74fc66c69d 100644 --- a/verifiers/envs/experimental/composable/swe_debug_env.py +++ b/verifiers/envs/experimental/composable/swe_debug_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Deprecated SWE-specific debug environment wrappers.""" from typing import Any diff --git a/verifiers/envs/experimental/composable/task.py b/verifiers/envs/experimental/composable/task.py index 950812c276..36251a80e3 100644 --- a/verifiers/envs/experimental/composable/task.py +++ b/verifiers/envs/experimental/composable/task.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Task, TaskSet, SandboxTaskSet, and SandboxSpec — WHAT to solve. A **Task** is a single, fully-bound problem instance. diff --git a/verifiers/envs/experimental/composable/tasksets/__init__.py b/verifiers/envs/experimental/composable/tasksets/__init__.py index b20dee438d..e7f523c2a6 100644 --- a/verifiers/envs/experimental/composable/tasksets/__init__.py +++ b/verifiers/envs/experimental/composable/tasksets/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.envs.experimental.composable.tasksets.swe.swe_tasksets import ( make_multiswe_taskset, make_openswe_taskset, diff --git a/verifiers/envs/experimental/composable/tasksets/cp/__init__.py b/verifiers/envs/experimental/composable/tasksets/cp/__init__.py index 7fdcc754f8..3de0444569 100644 --- a/verifiers/envs/experimental/composable/tasksets/cp/__init__.py +++ b/verifiers/envs/experimental/composable/tasksets/cp/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from .cp_task import CPRubric, CPTaskSet __all__ = ["CPTaskSet", "CPRubric"] diff --git a/verifiers/envs/experimental/composable/tasksets/cp/cp_task.py b/verifiers/envs/experimental/composable/tasksets/cp/cp_task.py index fe1e6886b4..22c3655740 100644 --- a/verifiers/envs/experimental/composable/tasksets/cp/cp_task.py +++ b/verifiers/envs/experimental/composable/tasksets/cp/cp_task.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Competitive Programming TaskSet. Usage:: diff --git a/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py b/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py index 4393633167..ca39b3e536 100644 --- a/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py +++ b/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + # modifed from https://github.com/hendrycks/apps/blob/main/eval/testing_util.py # https://github.com/NovaSky-AI/SkyThought/blob/main/skythought/skythought_evals/tasks/taco/taco_util.py import asyncio diff --git a/verifiers/envs/experimental/composable/tasksets/harbor/__init__.py b/verifiers/envs/experimental/composable/tasksets/harbor/__init__.py index 6cc326d9ac..f52b552938 100644 --- a/verifiers/envs/experimental/composable/tasksets/harbor/__init__.py +++ b/verifiers/envs/experimental/composable/tasksets/harbor/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from .harbor import ( HarborDatasetRubric, HarborDatasetTaskSet, diff --git a/verifiers/envs/experimental/composable/tasksets/harbor/harbor.py b/verifiers/envs/experimental/composable/tasksets/harbor/harbor.py index 55403ac215..6c5edeb282 100644 --- a/verifiers/envs/experimental/composable/tasksets/harbor/harbor.py +++ b/verifiers/envs/experimental/composable/tasksets/harbor/harbor.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import tarfile diff --git a/verifiers/envs/experimental/composable/tasksets/harbor/terminal_lego.py b/verifiers/envs/experimental/composable/tasksets/harbor/terminal_lego.py index ba61acab11..5c588ba679 100644 --- a/verifiers/envs/experimental/composable/tasksets/harbor/terminal_lego.py +++ b/verifiers/envs/experimental/composable/tasksets/harbor/terminal_lego.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Terminal-Lego taskset built on the composable Harbor task layout.""" import contextlib diff --git a/verifiers/envs/experimental/composable/tasksets/math/math_task.py b/verifiers/envs/experimental/composable/tasksets/math/math_task.py index 92c4169ad4..be9664e91a 100644 --- a/verifiers/envs/experimental/composable/tasksets/math/math_task.py +++ b/verifiers/envs/experimental/composable/tasksets/math/math_task.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Math TaskSet — QA-style math problems with sandbox-based scoring. Usage:: diff --git a/verifiers/envs/experimental/composable/tasksets/swe/__init__.py b/verifiers/envs/experimental/composable/tasksets/swe/__init__.py index 0c2828a041..960f60dff3 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/__init__.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from .swe_tasksets import ( make_multiswe_taskset, make_openswe_taskset, diff --git a/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/taskset.py index 72f7e4ea8f..62938903b8 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging import shlex import tempfile diff --git a/verifiers/envs/experimental/composable/tasksets/swe/openswe/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/openswe/taskset.py index 251e4e8697..4211d600b5 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/openswe/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/openswe/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging import re import tempfile diff --git a/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/taskset.py index a514a21e28..095d8e4529 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import tempfile diff --git a/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py index bf850ec8c3..da199820fb 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Scale-SWE TaskSet. Scale-SWE (https://huggingface.co/datasets/AweAI-Team/Scale-SWE) is a diff --git a/verifiers/envs/experimental/composable/tasksets/swe/shared/test_patch.py b/verifiers/envs/experimental/composable/tasksets/swe/shared/test_patch.py index f8c619d91e..352fdff5e4 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/shared/test_patch.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/shared/test_patch.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Revert-agent-test-edits + re-apply-test_patch helper. Mirrors the canonical SWE-bench pattern used by upstream's harness before diff --git a/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/taskset.py index e7c1d8259b..354e4dd083 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import re diff --git a/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/taskset.py index baf75dea5c..73eb0d0947 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import re diff --git a/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/taskset.py index 563c480e1f..d2915e2276 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + """SWE-rebench-V2 TaskSet. SWE-rebench-V2 (https://huggingface.co/datasets/nebius/SWE-rebench-V2) is a diff --git a/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/taskset.py index 7c99228a6a..9ee23d74c4 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + """SWE-Smith multilingual TaskSet. SWE-Smith (https://github.com/SWE-bench/SWE-smith) is a synthetic bug-fix diff --git a/verifiers/envs/experimental/gym_env.py b/verifiers/envs/experimental/gym_env.py index ff9c8f1e1b..8e048f42fd 100644 --- a/verifiers/envs/experimental/gym_env.py +++ b/verifiers/envs/experimental/gym_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + from collections.abc import Callable from typing import Any, Protocol, TypeAlias, cast diff --git a/verifiers/envs/experimental/harbor_env/__init__.py b/verifiers/envs/experimental/harbor_env/__init__.py index 7db51e4d2f..5ae8bf0f12 100644 --- a/verifiers/envs/experimental/harbor_env/__init__.py +++ b/verifiers/envs/experimental/harbor_env/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from .env import HarborEnv from .mcp import ( NETWORK_TRANSPORTS, diff --git a/verifiers/envs/experimental/harbor_env/env.py b/verifiers/envs/experimental/harbor_env/env.py index 9cd256dd00..c591d98389 100644 --- a/verifiers/envs/experimental/harbor_env/env.py +++ b/verifiers/envs/experimental/harbor_env/env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import tarfile diff --git a/verifiers/envs/experimental/mcp_env.py b/verifiers/envs/experimental/mcp_env.py index 49a94eb257..d1e4caa1cc 100644 --- a/verifiers/envs/experimental/mcp_env.py +++ b/verifiers/envs/experimental/mcp_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import logging import threading diff --git a/verifiers/envs/experimental/opencode_env.py b/verifiers/envs/experimental/opencode_env.py index 5108ff5d0a..b50b2043fe 100644 --- a/verifiers/envs/experimental/opencode_env.py +++ b/verifiers/envs/experimental/opencode_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import json import logging diff --git a/verifiers/envs/experimental/opencode_rlm_env.py b/verifiers/envs/experimental/opencode_rlm_env.py index a88a7b3583..fc9f7fb3b2 100644 --- a/verifiers/envs/experimental/opencode_rlm_env.py +++ b/verifiers/envs/experimental/opencode_rlm_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """ OpenCode RLM Environment. diff --git a/verifiers/envs/experimental/sandbox_mixin.py b/verifiers/envs/experimental/sandbox_mixin.py index 3623b33601..57e582b8dc 100644 --- a/verifiers/envs/experimental/sandbox_mixin.py +++ b/verifiers/envs/experimental/sandbox_mixin.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import io import logging diff --git a/verifiers/envs/experimental/utils/file_locks.py b/verifiers/envs/experimental/utils/file_locks.py index 5aa18cb25f..5f61559b38 100644 --- a/verifiers/envs/experimental/utils/file_locks.py +++ b/verifiers/envs/experimental/utils/file_locks.py @@ -1,3 +1,5 @@ +# ruff: noqa + from collections.abc import Iterator from contextlib import contextmanager import fcntl diff --git a/verifiers/envs/experimental/utils/git_checkout_cache.py b/verifiers/envs/experimental/utils/git_checkout_cache.py index 45a481fa17..25ef837b18 100644 --- a/verifiers/envs/experimental/utils/git_checkout_cache.py +++ b/verifiers/envs/experimental/utils/git_checkout_cache.py @@ -1,3 +1,5 @@ +# ruff: noqa + import fcntl import hashlib import logging diff --git a/verifiers/envs/integrations/README.md b/verifiers/envs/integrations/README.md index 4e08187114..07cde8e003 100644 --- a/verifiers/envs/integrations/README.md +++ b/verifiers/envs/integrations/README.md @@ -32,9 +32,18 @@ from datasets import Dataset import verifiers as vf # Create your dataset -dataset = Dataset.from_list([ - {"prompt": [{"role": "user", "content": "Navigate to example.com and find the main heading"}]}, -]) +dataset = Dataset.from_list( + [ + { + "prompt": [ + { + "role": "user", + "content": "Navigate to example.com and find the main heading", + } + ] + }, + ] +) # Create a rubric rubric = vf.Rubric(funcs=[my_reward_func]) diff --git a/verifiers/envs/integrations/browser_env/modes/__init__.py b/verifiers/envs/integrations/browser_env/modes/__init__.py index 74d5850f14..04ab6db5ec 100644 --- a/verifiers/envs/integrations/browser_env/modes/__init__.py +++ b/verifiers/envs/integrations/browser_env/modes/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Browser mode implementations.""" import warnings diff --git a/verifiers/envs/integrations/browser_env/modes/base.py b/verifiers/envs/integrations/browser_env/modes/base.py index e431122483..c09c70b99b 100644 --- a/verifiers/envs/integrations/browser_env/modes/base.py +++ b/verifiers/envs/integrations/browser_env/modes/base.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Protocol defining the interface for browser mode implementations.""" from typing import Protocol, Any, TYPE_CHECKING diff --git a/verifiers/envs/integrations/browser_env/modes/cua_mode.py b/verifiers/envs/integrations/browser_env/modes/cua_mode.py index 9e0ca3df4d..f4a43ba7d4 100644 --- a/verifiers/envs/integrations/browser_env/modes/cua_mode.py +++ b/verifiers/envs/integrations/browser_env/modes/cua_mode.py @@ -1,3 +1,5 @@ +# ruff: noqa + """CUA-based browser mode supporting both local HTTP and sandbox execution.""" import asyncio diff --git a/verifiers/envs/integrations/browser_env/modes/dom_mode.py b/verifiers/envs/integrations/browser_env/modes/dom_mode.py index 8f1a2ba1d5..102e420636 100644 --- a/verifiers/envs/integrations/browser_env/modes/dom_mode.py +++ b/verifiers/envs/integrations/browser_env/modes/dom_mode.py @@ -1,3 +1,5 @@ +# ruff: noqa + """DOM-based browser mode using Stagehand SDK.""" import asyncio diff --git a/verifiers/envs/integrations/openenv_env.py b/verifiers/envs/integrations/openenv_env.py index ccc47783ac..d1172b6b86 100644 --- a/verifiers/envs/integrations/openenv_env.py +++ b/verifiers/envs/integrations/openenv_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import inspect import json diff --git a/verifiers/envs/integrations/reasoninggym_env.py b/verifiers/envs/integrations/reasoninggym_env.py index 942eae5c6c..d6a79194c2 100644 --- a/verifiers/envs/integrations/reasoninggym_env.py +++ b/verifiers/envs/integrations/reasoninggym_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import List, Tuple from datasets import Dataset diff --git a/verifiers/envs/integrations/textarena_env.py b/verifiers/envs/integrations/textarena_env.py index ea1029ca59..7a438b7437 100644 --- a/verifiers/envs/integrations/textarena_env.py +++ b/verifiers/envs/integrations/textarena_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import logging import random diff --git a/verifiers/envs/multiturn_env.py b/verifiers/envs/multiturn_env.py index 40d1aa3c9a..a8e7329fca 100644 --- a/verifiers/envs/multiturn_env.py +++ b/verifiers/envs/multiturn_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import logging import time diff --git a/verifiers/envs/sandbox_env.py b/verifiers/envs/sandbox_env.py index 36031d65c9..27b7af4bde 100644 --- a/verifiers/envs/sandbox_env.py +++ b/verifiers/envs/sandbox_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging import time from typing import Any, TypedDict, cast diff --git a/verifiers/envs/stateful_tool_env.py b/verifiers/envs/stateful_tool_env.py index cb44555a41..92de0c7c3a 100644 --- a/verifiers/envs/stateful_tool_env.py +++ b/verifiers/envs/stateful_tool_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import inspect import json from abc import abstractmethod diff --git a/verifiers/envs/tool_env.py b/verifiers/envs/tool_env.py index ee1ed07e7d..ed057b5fa0 100644 --- a/verifiers/envs/tool_env.py +++ b/verifiers/envs/tool_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json from typing import Callable, cast diff --git a/verifiers/errors.py b/verifiers/errors.py index e725580e48..1f3dd16d45 100644 --- a/verifiers/errors.py +++ b/verifiers/errors.py @@ -1,3 +1,6 @@ +# ruff: noqa + + class Error(Exception): """Base class for all errors.""" diff --git a/verifiers/gepa/__init__.py b/verifiers/gepa/__init__.py index c20cb724e7..7c53ee2570 100644 --- a/verifiers/gepa/__init__.py +++ b/verifiers/gepa/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.gepa.adapter import VerifiersGEPAAdapter, make_reflection_lm from verifiers.gepa.gepa_utils import save_gepa_results from verifiers.gepa.config import GEPAConfig diff --git a/verifiers/gepa/adapter.py b/verifiers/gepa/adapter.py index 869791423c..cd8e72412d 100644 --- a/verifiers/gepa/adapter.py +++ b/verifiers/gepa/adapter.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import logging from dataclasses import dataclass, field diff --git a/verifiers/gepa/gepa_utils.py b/verifiers/gepa/gepa_utils.py index 34db6d7f36..06be0fcefc 100644 --- a/verifiers/gepa/gepa_utils.py +++ b/verifiers/gepa/gepa_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """ Simple artifact saving for GEPA optimization. diff --git a/verifiers/parsers/maybe_think_parser.py b/verifiers/parsers/maybe_think_parser.py index aed453e682..7916183b74 100644 --- a/verifiers/parsers/maybe_think_parser.py +++ b/verifiers/parsers/maybe_think_parser.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import Callable from verifiers.parsers.parser import Parser diff --git a/verifiers/parsers/parser.py b/verifiers/parsers/parser.py index 9e9dee68bb..c8a9bc0720 100644 --- a/verifiers/parsers/parser.py +++ b/verifiers/parsers/parser.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging from typing import Any, Callable diff --git a/verifiers/parsers/think_parser.py b/verifiers/parsers/think_parser.py index e65b954f03..19abcf15de 100644 --- a/verifiers/parsers/think_parser.py +++ b/verifiers/parsers/think_parser.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import Callable from verifiers.parsers.parser import Parser diff --git a/verifiers/parsers/xml_parser.py b/verifiers/parsers/xml_parser.py index 470c6debed..19e405786e 100644 --- a/verifiers/parsers/xml_parser.py +++ b/verifiers/parsers/xml_parser.py @@ -1,3 +1,5 @@ +# ruff: noqa + import re from types import SimpleNamespace from typing import Any, Callable diff --git a/verifiers/rubrics/experimental/hybrid_math_rubric.py b/verifiers/rubrics/experimental/hybrid_math_rubric.py index 0d4fdd7aff..ddec7212a7 100644 --- a/verifiers/rubrics/experimental/hybrid_math_rubric.py +++ b/verifiers/rubrics/experimental/hybrid_math_rubric.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio from openai import AsyncOpenAI diff --git a/verifiers/rubrics/judge_rubric.py b/verifiers/rubrics/judge_rubric.py index 9a8b737062..f915c8aaa1 100644 --- a/verifiers/rubrics/judge_rubric.py +++ b/verifiers/rubrics/judge_rubric.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import Any from openai import APIError, APITimeoutError, AsyncOpenAI, RateLimitError diff --git a/verifiers/rubrics/math_rubric.py b/verifiers/rubrics/math_rubric.py index dfa7377e87..323a442890 100644 --- a/verifiers/rubrics/math_rubric.py +++ b/verifiers/rubrics/math_rubric.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import os import sys diff --git a/verifiers/rubrics/rubric.py b/verifiers/rubrics/rubric.py index 067742005f..6c45d7a4f3 100644 --- a/verifiers/rubrics/rubric.py +++ b/verifiers/rubrics/rubric.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import inspect import logging diff --git a/verifiers/scripts/build.py b/verifiers/scripts/build.py index 07bf55aa39..c02e795491 100644 --- a/verifiers/scripts/build.py +++ b/verifiers/scripts/build.py @@ -1,3 +1,5 @@ +# ruff: noqa + import argparse import ast import json diff --git a/verifiers/scripts/eval.py b/verifiers/scripts/eval.py index 01bdd77496..aa5b946349 100644 --- a/verifiers/scripts/eval.py +++ b/verifiers/scripts/eval.py @@ -1,3 +1,5 @@ +# ruff: noqa + import os from verifiers.utils.path_utils import ( diff --git a/verifiers/scripts/gepa.py b/verifiers/scripts/gepa.py index 16944e60d3..053c35e21a 100644 --- a/verifiers/scripts/gepa.py +++ b/verifiers/scripts/gepa.py @@ -1,3 +1,5 @@ +# ruff: noqa + """ GEPA prompt optimization CLI. diff --git a/verifiers/scripts/install.py b/verifiers/scripts/install.py index 37c9b80e53..670faa8d0a 100644 --- a/verifiers/scripts/install.py +++ b/verifiers/scripts/install.py @@ -1,3 +1,5 @@ +# ruff: noqa + import argparse from verifiers.utils.install_utils import ( diff --git a/verifiers/scripts/setup.py b/verifiers/scripts/setup.py index 6408f1d4d3..742d1d151e 100644 --- a/verifiers/scripts/setup.py +++ b/verifiers/scripts/setup.py @@ -1,3 +1,5 @@ +# ruff: noqa + import sys from collections.abc import Sequence diff --git a/verifiers/serve/__init__.py b/verifiers/serve/__init__.py index c3b13c6754..c624081afe 100644 --- a/verifiers/serve/__init__.py +++ b/verifiers/serve/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.serve.client.env_client import EnvClient from verifiers.serve.client.zmq_env_client import ZMQEnvClient from verifiers.serve.server import EnvRouter, EnvServer, EnvWorker, ZMQEnvServer diff --git a/verifiers/serve/client/env_client.py b/verifiers/serve/client/env_client.py index 8649fb2460..ecb7ce4d10 100644 --- a/verifiers/serve/client/env_client.py +++ b/verifiers/serve/client/env_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging from abc import ABC, abstractmethod diff --git a/verifiers/serve/client/zmq_env_client.py b/verifiers/serve/client/zmq_env_client.py index fd7337f59a..08c9e92d45 100644 --- a/verifiers/serve/client/zmq_env_client.py +++ b/verifiers/serve/client/zmq_env_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import threading import time diff --git a/verifiers/serve/server/env_router.py b/verifiers/serve/server/env_router.py index a1dd5a4a7b..cb06d96093 100644 --- a/verifiers/serve/server/env_router.py +++ b/verifiers/serve/server/env_router.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Worker pool manager with IPC dispatch. Owns per-worker PUSH sockets and shared response/stats PULL sockets. diff --git a/verifiers/serve/server/env_worker.py b/verifiers/serve/server/env_worker.py index f5e3c9543b..cbdb519356 100644 --- a/verifiers/serve/server/env_worker.py +++ b/verifiers/serve/server/env_worker.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Environment worker. Owns a single environment instance, a client cache, and three ZMQ diff --git a/verifiers/serve/types.py b/verifiers/serve/types.py index b5b3e6c5b9..85b898885d 100644 --- a/verifiers/serve/types.py +++ b/verifiers/serve/types.py @@ -1,3 +1,5 @@ +# ruff: noqa + from asyncio import Future from enum import Enum from typing import Annotated, Literal, TypeAlias, TypeVar diff --git a/verifiers/types.py b/verifiers/types.py index 35eeefdeb6..17f6c1e04c 100644 --- a/verifiers/types.py +++ b/verifiers/types.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import sys import time diff --git a/verifiers/utils/async_utils.py b/verifiers/utils/async_utils.py index 1195599dab..a22df5fd69 100644 --- a/verifiers/utils/async_utils.py +++ b/verifiers/utils/async_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import inspect import logging diff --git a/verifiers/utils/client_utils.py b/verifiers/utils/client_utils.py index d1c9b62e2a..acd904e46d 100644 --- a/verifiers/utils/client_utils.py +++ b/verifiers/utils/client_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import os diff --git a/verifiers/utils/data_utils.py b/verifiers/utils/data_utils.py index 8840f3b184..6aa127d6d4 100644 --- a/verifiers/utils/data_utils.py +++ b/verifiers/utils/data_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + # NOTE: Helper functions for example datasets. Not intended for core functionality. import random diff --git a/verifiers/utils/display_utils.py b/verifiers/utils/display_utils.py index 7f9b22769c..3314d8b488 100644 --- a/verifiers/utils/display_utils.py +++ b/verifiers/utils/display_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """ Shared utilities for Rich-based terminal displays. diff --git a/verifiers/utils/env_config_utils.py b/verifiers/utils/env_config_utils.py index 51e1114fd0..77cd0e6f1d 100644 --- a/verifiers/utils/env_config_utils.py +++ b/verifiers/utils/env_config_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + from collections.abc import Mapping from typing import cast diff --git a/verifiers/utils/env_utils.py b/verifiers/utils/env_utils.py index 06318cd9ab..fa81bf0c46 100644 --- a/verifiers/utils/env_utils.py +++ b/verifiers/utils/env_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import importlib import inspect import logging diff --git a/verifiers/utils/error_utils.py b/verifiers/utils/error_utils.py index 37ac1a4058..fda0ab89a6 100644 --- a/verifiers/utils/error_utils.py +++ b/verifiers/utils/error_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import Callable, TypeGuard, cast import verifiers as vf diff --git a/verifiers/utils/eval_display.py b/verifiers/utils/eval_display.py index 0f2c34c186..8df5101de3 100644 --- a/verifiers/utils/eval_display.py +++ b/verifiers/utils/eval_display.py @@ -1,3 +1,5 @@ +# ruff: noqa + """ Rich-based display for live multi-environment evaluation. diff --git a/verifiers/utils/eval_utils.py b/verifiers/utils/eval_utils.py index f33006dea4..e1cdabf72c 100644 --- a/verifiers/utils/eval_utils.py +++ b/verifiers/utils/eval_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import itertools import json diff --git a/verifiers/utils/heartbeat.py b/verifiers/utils/heartbeat.py index de581f0f50..1cdde2df7a 100644 --- a/verifiers/utils/heartbeat.py +++ b/verifiers/utils/heartbeat.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging import aiohttp diff --git a/verifiers/utils/install_utils.py b/verifiers/utils/install_utils.py index 018e8a3bf1..011e05ba16 100644 --- a/verifiers/utils/install_utils.py +++ b/verifiers/utils/install_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import importlib import logging import subprocess diff --git a/verifiers/utils/interception_utils.py b/verifiers/utils/interception_utils.py index a220e0f734..a50d435f02 100644 --- a/verifiers/utils/interception_utils.py +++ b/verifiers/utils/interception_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Utilities for intercepting API calls from agents running in sandboxes.""" import asyncio diff --git a/verifiers/utils/logging_utils.py b/verifiers/utils/logging_utils.py index 0f88b8d73c..edc1ceb488 100644 --- a/verifiers/utils/logging_utils.py +++ b/verifiers/utils/logging_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import sys diff --git a/verifiers/utils/message_utils.py b/verifiers/utils/message_utils.py index f6fc717c45..7a7cbb86d4 100644 --- a/verifiers/utils/message_utils.py +++ b/verifiers/utils/message_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import re diff --git a/verifiers/utils/metric_utils.py b/verifiers/utils/metric_utils.py index 41adea17e6..ec08385554 100644 --- a/verifiers/utils/metric_utils.py +++ b/verifiers/utils/metric_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import math from abc import ABC, abstractmethod from collections import defaultdict diff --git a/verifiers/utils/process_utils.py b/verifiers/utils/process_utils.py index 3fdc8dbd74..f241ddbf99 100644 --- a/verifiers/utils/process_utils.py +++ b/verifiers/utils/process_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Process lifecycle utilities.""" import logging diff --git a/verifiers/utils/save_utils.py b/verifiers/utils/save_utils.py index 8f59e8e01d..8f2acd4088 100644 --- a/verifiers/utils/save_utils.py +++ b/verifiers/utils/save_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import os diff --git a/verifiers/utils/thread_utils.py b/verifiers/utils/thread_utils.py index e84de38196..4d516134f5 100644 --- a/verifiers/utils/thread_utils.py +++ b/verifiers/utils/thread_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import logging import threading diff --git a/verifiers/utils/threaded_sandbox_client.py b/verifiers/utils/threaded_sandbox_client.py index 25df03fc04..652c4cc97c 100644 --- a/verifiers/utils/threaded_sandbox_client.py +++ b/verifiers/utils/threaded_sandbox_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import functools import logging diff --git a/verifiers/utils/usage_utils.py b/verifiers/utils/usage_utils.py index c8e3921b04..a4dbbed351 100644 --- a/verifiers/utils/usage_utils.py +++ b/verifiers/utils/usage_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + from collections.abc import Mapping, Sequence from types import MappingProxyType diff --git a/verifiers/utils/version_utils.py b/verifiers/utils/version_utils.py index 76e29f2c22..0ec6e56fb5 100644 --- a/verifiers/utils/version_utils.py +++ b/verifiers/utils/version_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Utilities for detecting verifiers and environment version/commit info.""" import importlib.metadata diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 41b3512cb3..ce5e6a6756 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -3,6 +3,7 @@ from pydantic_config import BaseConfig from verifiers.v1.acp import ACP +from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent from verifiers.v1.clients import ( BaseClientConfig, Client, @@ -12,9 +13,7 @@ TrainClientConfig, resolve_client, ) -from verifiers.v1.decorators import metric, reward, stop, tool from verifiers.v1.configs.agent import AgentConfig -from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent from verifiers.v1.configs.cli.env import ( ElasticPoolConfig, EnvServerConfig, @@ -22,8 +21,15 @@ pool_serve_kwargs, ) from verifiers.v1.configs.env import EnvConfig, default_agent_harness +from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.judge import JudgeConfig, Judges +from verifiers.v1.configs.retries import RetryConfig +from verifiers.v1.configs.task import TaskConfig +from verifiers.v1.configs.taskset import TasksetConfig +from verifiers.v1.decorators import metric, reward, stop, tool from verifiers.v1.env import Env from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig +from verifiers.v1.episode import Episode, WireEpisode from verifiers.v1.errors import ( EnvError, HarnessError, @@ -35,21 +41,19 @@ ToolsetError, TunnelError, ) -from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.graph import MessageNode from verifiers.v1.harness import Harness -from verifiers.v1.configs.judge import JudgeConfig, Judges from verifiers.v1.judge import Judge, JudgeResponse, JudgeView from verifiers.v1.judges import ( + Criterion, ReferenceJudge, ReferenceJudgeConfig, - Criterion, RubricJudge, RubricJudgeConfig, ) from verifiers.v1.loaders import ( default_harness_id, env_config_type, - resolve_env_config, environment_class, harness_config_type, import_environment, @@ -61,22 +65,14 @@ load_harness, load_judge, load_taskset, + resolve_env_config, task_type, taskset_config_type, ) -from verifiers.v1.scoring import ( - compare_stdout_results as compare_stdout_results, - extract_boxed_answer as extract_boxed_answer, - parse_judge_choice as parse_judge_choice, - parse_pytest_outcomes as parse_pytest_outcomes, - read_answer_file_or_last_reply as read_answer_file_or_last_reply, - verify_boxed_math_answer as verify_boxed_math_answer, -) -from verifiers.v1.configs.retries import RetryConfig -from verifiers.v1.utils.git import ( - PATCH_CAP_BYTES as PATCH_CAP_BYTES, - capture_patch as capture_patch, - resolve_head as resolve_head, +from verifiers.v1.mcp import ( + SharedToolsetConfig, + Toolset, + ToolsetConfig, ) from verifiers.v1.runtimes import ( DockerConfig, @@ -87,18 +83,27 @@ RuntimeInfo, SubprocessConfig, ) +from verifiers.v1.scoring import ( + compare_stdout_results as compare_stdout_results, +) +from verifiers.v1.scoring import ( + extract_boxed_answer as extract_boxed_answer, +) +from verifiers.v1.scoring import ( + parse_judge_choice as parse_judge_choice, +) +from verifiers.v1.scoring import ( + parse_pytest_outcomes as parse_pytest_outcomes, +) +from verifiers.v1.scoring import ( + read_answer_file_or_last_reply as read_answer_file_or_last_reply, +) +from verifiers.v1.scoring import ( + verify_boxed_math_answer as verify_boxed_math_answer, +) from verifiers.v1.state import State, StateT -from verifiers.v1.configs.task import TaskConfig from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout, WireTaskData -from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.taskset import Taskset -from verifiers.v1.mcp import ( - Toolset, - SharedToolsetConfig, - ToolsetConfig, -) -from verifiers.v1.graph import MessageNode -from verifiers.v1.episode import Episode, WireEpisode from verifiers.v1.trace import ( TRACE_VERSION, AgentInfo, @@ -119,9 +124,9 @@ WireTrace, ) from verifiers.v1.types import ( + ID, AssistantMessage, ContentPart, - ID, ImageUrlContentPart, ImageUrlSource, KeptTokens, @@ -136,13 +141,22 @@ TextContentPart, Tool, ToolCall, - TurnTokens, ToolMessage, + TurnTokens, Usage, UserMessage, ) +from verifiers.v1.utils.git import ( + PATCH_CAP_BYTES as PATCH_CAP_BYTES, +) +from verifiers.v1.utils.git import ( + capture_patch as capture_patch, +) +from verifiers.v1.utils.git import ( + resolve_head as resolve_head, +) -__all__ = [ +__all__ = [ # noqa: RUF022 - grouped by public API area # types "ID", "AssistantMessage", diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index d7ffda028f..f99158f3af 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -9,19 +9,18 @@ import asyncio import logging -from collections.abc import Callable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Iterator, Mapping from contextlib import asynccontextmanager, nullcontext from dataclasses import dataclass -from typing import AsyncIterator +from typing import Self - -from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig from verifiers.v1.clients import ( Client, EvalClientConfig, ModelContext, resolve_client, ) +from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig from verifiers.v1.harness import Harness from verifiers.v1.interception import Interception, InterceptionServer from verifiers.v1.mcp import SharedToolServer @@ -96,7 +95,7 @@ def _check_borrowed_placement( if task.data.image is None: return if isinstance(runtime.config, SubprocessConfig): - raise ValueError( + raise TypeError( f"task {task.data.idx!r} requires image {task.data.image!r}, but the " "borrowed runtime is subprocess-backed (no container); borrow a container " "box (e.g. agent.provision(task)) or drop the task's image" @@ -170,9 +169,8 @@ async def turn(self, message: str | Messages | None = None) -> Segment: prompted task speaks FIRST: take its opening reply with a bare `turn()` before answering. A `terminated` segment means the run ended instead of answering (the message went unconsumed).""" - async with self._lock: - async with self._gate or nullcontext(): - return await self._turn(message) + async with self._lock, self._gate or nullcontext(): + return await self._turn(message) async def _turn(self, message: str | Messages | None) -> Segment: if self._run.closed: @@ -220,11 +218,10 @@ async def _turn(self, message: str | Messages | None) -> Segment: async def close(self) -> Trace: """End the exchange and finish the rollout (idempotent): scoring and hooks run, then the finished trace returns (also on `interaction.trace`).""" - async with self._lock: - async with self._gate or nullcontext(): - if not self._run.closed and self._run.ok: - self.trace.stop("user_closed") - return await self._run.close() + async with self._lock, self._gate or nullcontext(): + if not self._run.closed and self._run.ok: + self.trace.stop("user_closed") + return await self._run.close() class Agent: @@ -290,7 +287,7 @@ def __init__( self._server: InterceptionServer | None = None self._warned_resources: set[tuple[str, str]] = set() - async def __aenter__(self) -> "Agent": + async def __aenter__(self) -> Self: if self._entered: raise RuntimeError("Agent is already entered; enter it once and share it") if self._closed: @@ -528,34 +525,34 @@ def _rollout_params( if self.timeout.rollout is not None else task.data.timeout.harness ) - return dict( - agent_config=self.config, - harness=self.harness, - ctx=self.ctx, - runtime_config=runtime_config, - setup_timeout=( + return { + "agent_config": self.config, + "harness": self.harness, + "ctx": self.ctx, + "runtime_config": runtime_config, + "setup_timeout": ( self.timeout.setup if self.timeout.setup is not None else task.data.timeout.setup ), - harness_timeout=cap_remote_harness_timeout( + "harness_timeout": cap_remote_harness_timeout( harness_timeout, runtime_config, task ), - finalize_timeout=( + "finalize_timeout": ( self.timeout.finalize if self.timeout.finalize is not None else task.data.timeout.finalize ), - scoring_timeout=( + "scoring_timeout": ( self.timeout.scoring if self.timeout.scoring is not None else task.data.timeout.scoring ), - limits=self.limits, - shared_tools=shared_tools, - interception=self._interception_for(run_is_local, task, shared_tools), - runtime=runtime, - ) + "limits": self.limits, + "shared_tools": shared_tools, + "interception": self._interception_for(run_is_local, task, shared_tools), + "runtime": runtime, + } @asynccontextmanager async def provision(self, task: Task | None = None) -> AsyncIterator[Runtime]: @@ -719,10 +716,8 @@ class Agents: field becomes an `Agent` under the field's name (`agents.solver`).""" def __init__(self, config, make: MakeAgent | None = None) -> None: - if make is None: - make = lambda _, spec: make_agent(spec) # noqa: E731 self._agents: dict[str, Agent] = { - name: make(name, value) + name: make_agent(value) if make is None else make(name, value) for name, value in agent_config_fields(config).items() } diff --git a/verifiers/v1/cli/dashboard/eval.py b/verifiers/v1/cli/dashboard/eval.py index 67f2ab7f02..f84392d40a 100644 --- a/verifiers/v1/cli/dashboard/eval.py +++ b/verifiers/v1/cli/dashboard/eval.py @@ -12,10 +12,9 @@ from rich.table import Table from rich.text import Text +from verifiers.utils.pricing_utils import format_cost_usd from verifiers.v1.cli.dashboard.base import live_view from verifiers.v1.cli.output import output_path -from verifiers.v1.utils.install import env_name -from verifiers.v1.utils.interrupt import cleaning_up from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.env import RunSlot from verifiers.v1.trace import Trace @@ -26,7 +25,8 @@ format_override, format_time, ) -from verifiers.utils.pricing_utils import format_cost_usd +from verifiers.v1.utils.install import env_name +from verifiers.v1.utils.interrupt import cleaning_up if TYPE_CHECKING: from verifiers.v1.push import PushState diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py index eed30f8067..a4893cbabe 100644 --- a/verifiers/v1/cli/debug.py +++ b/verifiers/v1/cli/debug.py @@ -24,11 +24,11 @@ ) from verifiers.v1.configs.cli.debug import DebugConfig from verifiers.v1.decorators import invoke -from verifiers.v1.utils.compile import resolve_runtime_config from verifiers.v1.runtimes import ProgramResult, Runtime, make_runtime from verifiers.v1.state import state_cls from verifiers.v1.task import Task from verifiers.v1.trace import AgentInfo, Error, Trace, TraceTask +from verifiers.v1.utils.compile import resolve_runtime_config from verifiers.v1.utils.interrupt import install_interrupt from verifiers.v1.utils.logging import setup_logging @@ -165,7 +165,7 @@ async def run_timed( start = time.time() try: result = await asyncio.wait_for(action, config.timeout.total) - except Exception as e: + except Exception as e: # noqa: BLE001 - report any debug action failure return error_info(e, start, config.timeout.total, "debug action") return result_info(result, start) @@ -250,7 +250,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]: except asyncio.CancelledError as e: cancelled = True record_debug_error(trace, debug, e, setup_timeout, config.timeout.total) - except Exception as e: + except Exception as e: # noqa: BLE001 - persist any framework failure on the trace record_debug_error(trace, debug, e, setup_timeout, config.timeout.total) finally: trace.split_generation() diff --git a/verifiers/v1/cli/eval/main.py b/verifiers/v1/cli/eval/main.py index 682f76163a..0b3f66ae0f 100644 --- a/verifiers/v1/cli/eval/main.py +++ b/verifiers/v1/cli/eval/main.py @@ -7,8 +7,8 @@ from pydantic_config import cli import verifiers.v1 as vf -from verifiers.v1.utils.interrupt import install_interrupt -from verifiers.v1.utils.logging import setup_logging +from verifiers.v1.cli.eval.resume import load_resume_config, split_resume +from verifiers.v1.cli.eval.runner import run_eval from verifiers.v1.cli.output import output_path, write_config from verifiers.v1.cli.resolve import ( extract_id, @@ -17,9 +17,9 @@ references_config_file, with_positional_taskset, ) -from verifiers.v1.cli.eval.resume import load_resume_config, split_resume -from verifiers.v1.cli.eval.runner import run_eval from verifiers.v1.configs.cli.eval import EvalConfig +from verifiers.v1.utils.interrupt import install_interrupt +from verifiers.v1.utils.logging import setup_logging logger = logging.getLogger(__name__) diff --git a/verifiers/v1/cli/eval/resume.py b/verifiers/v1/cli/eval/resume.py index f12af98cf7..18bda68979 100644 --- a/verifiers/v1/cli/eval/resume.py +++ b/verifiers/v1/cli/eval/resume.py @@ -137,7 +137,8 @@ def parse(row: dict) -> Episode: episode = parse(row) if not verdict(episode): continue - except Exception: # malformed row: redo it + # A malformed row from any task/episode plugin is owed again. + except Exception: # noqa: BLE001, S112 continue good[key].append( (line if line.endswith(b"\n") else line + b"\n", episode) diff --git a/verifiers/v1/cli/eval/runner.py b/verifiers/v1/cli/eval/runner.py index 8eb1fb4e91..ca45870f8c 100644 --- a/verifiers/v1/cli/eval/runner.py +++ b/verifiers/v1/cli/eval/runner.py @@ -5,16 +5,16 @@ import logging import time -from verifiers.v1.clients import ModelContext, resolve_client -from verifiers.v1.configs.cli.eval import EvalConfig -from verifiers.v1.cli.eval import resume from verifiers.v1.cli.dashboard import dashboard +from verifiers.v1.cli.eval import resume from verifiers.v1.cli.output import ( append_episode, append_trace, output_path, save_config, ) +from verifiers.v1.clients import ModelContext, resolve_client +from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.env import Env, RunSlot from verifiers.v1.episode import Episode from verifiers.v1.trace import EvalRunInfo @@ -107,9 +107,9 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]: import multiprocessing as mp from functools import partial - from verifiers.v1.utils.logging import setup_logging from verifiers.v1.configs.cli.env import pool_serve_kwargs from verifiers.v1.serve import EnvClient, env_config_data, serve_env + from verifiers.v1.utils.logging import setup_logging legacy = config.is_legacy server_kwargs = ( diff --git a/verifiers/v1/cli/init.py b/verifiers/v1/cli/init.py index 3094573de3..b9c6e4b50c 100644 --- a/verifiers/v1/cli/init.py +++ b/verifiers/v1/cli/init.py @@ -17,7 +17,7 @@ def _names(name: str) -> tuple[str, str, str, str]: dash = name.strip().strip("/").replace("_", "-").lower() pkg = dash.replace("-", "_") - stem = pkg[:-3] if pkg.endswith("_v1") else pkg + stem = pkg.removesuffix("_v1") prefix = "".join(part[:1].upper() + part[1:] for part in stem.split("_") if part) if not prefix or not prefix[0].isalpha(): prefix = f"Env{prefix}" @@ -171,8 +171,10 @@ async def launch( def _readme(dash: str, pkg: str, *, add_tool: bool, add_harness: bool) -> str: layout = [ - f"- `{pkg}/taskset.py` — the task (`@reward` scoring + behavior) and the taskset: " - "`load` (data + prompts)." + ( + f"- `{pkg}/taskset.py` — the task (`@reward` scoring + behavior) and the taskset: " + "`load` (data + prompts)." + ) ] if add_tool: layout.append( diff --git a/verifiers/v1/cli/replay.py b/verifiers/v1/cli/replay.py index a535430221..4f8d16f725 100644 --- a/verifiers/v1/cli/replay.py +++ b/verifiers/v1/cli/replay.py @@ -27,10 +27,10 @@ save_config, write_config, ) +from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.configs.cli.replay import ReplayConfig from verifiers.v1.state import state_cls from verifiers.v1.task import Task, WireTaskData, task_data_cls -from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.trace import Trace from verifiers.v1.utils.interrupt import install_interrupt from verifiers.v1.utils.logging import setup_logging diff --git a/verifiers/v1/cli/serve.py b/verifiers/v1/cli/serve.py index 8269c89a64..6f5c5cdd5b 100644 --- a/verifiers/v1/cli/serve.py +++ b/verifiers/v1/cli/serve.py @@ -5,7 +5,6 @@ from pydantic_config import cli -from verifiers.v1.utils.logging import setup_logging from verifiers.v1.cli.resolve import ( extract_id, narrow_config, @@ -13,9 +12,10 @@ references_config_file, with_positional_taskset, ) -from verifiers.v1.configs.cli.serve import ServeConfig from verifiers.v1.configs.cli.env import pool_serve_kwargs +from verifiers.v1.configs.cli.serve import ServeConfig from verifiers.v1.serve import serve_env +from verifiers.v1.utils.logging import setup_logging USAGE = "usage: uv run serve [] [--env.id ] [--id (legacy)] [options] [@ file.toml]" diff --git a/verifiers/v1/cli/validate.py b/verifiers/v1/cli/validate.py index 7e446333d8..a889111736 100644 --- a/verifiers/v1/cli/validate.py +++ b/verifiers/v1/cli/validate.py @@ -20,11 +20,11 @@ ) from verifiers.v1.configs.cli.validate import ValidateConfig from verifiers.v1.decorators import invoke -from verifiers.v1.utils.compile import resolve_runtime_config from verifiers.v1.runtimes import make_runtime from verifiers.v1.state import state_cls from verifiers.v1.task import Task from verifiers.v1.trace import Trace, TraceTask +from verifiers.v1.utils.compile import resolve_runtime_config from verifiers.v1.utils.interrupt import install_interrupt from verifiers.v1.utils.logging import setup_logging @@ -103,7 +103,7 @@ async def _run_gold(task: Task, config: ValidateConfig) -> ResultRow: setup_timeout, ) valid = await asyncio.wait_for(task.validate(runtime), config.timeout.total) - except Exception as e: + except Exception as e: # noqa: BLE001 - validation reports plugin failures per task exc = e finally: try: @@ -138,7 +138,7 @@ async def _run_setup(task: Task, config: ValidateConfig) -> ResultRow: setup_timeout, ) valid = True - except Exception as e: + except Exception as e: # noqa: BLE001 - validation reports plugin failures per task exc = e finally: try: diff --git a/verifiers/v1/clients/__init__.py b/verifiers/v1/clients/__init__.py index 35728a3910..ab38dc7249 100644 --- a/verifiers/v1/clients/__init__.py +++ b/verifiers/v1/clients/__init__.py @@ -10,13 +10,13 @@ from verifiers.v1.clients.train import TrainClient __all__ = [ - "Client", - "ModelContext", "BaseClientConfig", + "Client", "ClientConfig", + "EvalClient", "EvalClientConfig", + "ModelContext", + "TrainClient", "TrainClientConfig", "resolve_client", - "EvalClient", - "TrainClient", ] diff --git a/verifiers/v1/clients/eval.py b/verifiers/v1/clients/eval.py index 6a7df3369c..2951bb23a8 100644 --- a/verifiers/v1/clients/eval.py +++ b/verifiers/v1/clients/eval.py @@ -12,8 +12,8 @@ change. Endpoint config (base url, api key, billing headers) comes from the client config. """ -from collections.abc import Mapping import re +from collections.abc import Mapping import httpx from pydantic import ValidationError diff --git a/verifiers/v1/clients/train.py b/verifiers/v1/clients/train.py index 6b7e7fad8e..785f05a923 100644 --- a/verifiers/v1/clients/train.py +++ b/verifiers/v1/clients/train.py @@ -13,9 +13,8 @@ from typing import Any from openai import AsyncOpenAI, OpenAIError -from renderers import RenderedTokens from renderers import OverlongPromptError as RendererOverlongPromptError -from renderers import RendererConfig +from renderers import RenderedTokens, RendererConfig from verifiers.v1.clients.client import SESSION_ID_HEADER, Client from verifiers.v1.dialects import FINISH_REASONS, ChatDialect, Dialect, parse_tools diff --git a/verifiers/v1/configs/cli/__init__.py b/verifiers/v1/configs/cli/__init__.py index 5e1de4beed..a5ee914a41 100644 --- a/verifiers/v1/configs/cli/__init__.py +++ b/verifiers/v1/configs/cli/__init__.py @@ -1,7 +1,7 @@ """The CLI entrypoints' run configs — each `uv run `'s parsed tree.""" -from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.configs.cli.debug import DebugConfig +from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.configs.cli.init import InitConfig from verifiers.v1.configs.cli.serve import ServeConfig from verifiers.v1.configs.cli.validate import ValidateConfig diff --git a/verifiers/v1/configs/cli/debug.py b/verifiers/v1/configs/cli/debug.py index 97df0bacde..3be8f1a031 100644 --- a/verifiers/v1/configs/cli/debug.py +++ b/verifiers/v1/configs/cli/debug.py @@ -7,8 +7,8 @@ from pydantic_config import BaseConfig from verifiers.v1.configs.cli.validate import CheckTimeoutConfig -from verifiers.v1.runtimes import DockerConfig, RuntimeConfig from verifiers.v1.configs.taskset import TasksetConfig +from verifiers.v1.runtimes import DockerConfig, RuntimeConfig class DebugConfig(BaseConfig): diff --git a/verifiers/v1/configs/cli/env.py b/verifiers/v1/configs/cli/env.py index ddba92462c..03f55e8efc 100644 --- a/verifiers/v1/configs/cli/env.py +++ b/verifiers/v1/configs/cli/env.py @@ -120,7 +120,7 @@ class EnvServerConfig(BaseConfig): """The environment — the run's `[env]` block: which env, its seed taskset, each agent, its knobs, and the run limits. Narrowed to the selected env's config class by the env id, else the taskset id.""" - pool: PoolConfig = ElasticPoolConfig() + pool: PoolConfig = Field(default_factory=ElasticPoolConfig) """Worker-pool sizing for the env server. `elastic` (default) starts at one worker and scales up on demand; `static` pre-spawns a fixed `num_workers`.""" # --- legacy (v0) backwards-compat ----------------------------------------- @@ -128,9 +128,9 @@ class by the env id, else the taskset id.""" """Classic (v0) env id (`name`, `org/name`, or `org/name@version` — installed from the hub on demand), loaded via `verifiers.load_environment` and run through the legacy bridge. Set this *instead of* `env.taskset` to run a v0 environment.""" - args: dict = {} + args: dict = Field(default_factory=dict) """Construction kwargs forwarded to `load_environment(id, **args)`.""" - extra_env_kwargs: dict = {} + extra_env_kwargs: dict = Field(default_factory=dict) """Post-load kwargs applied to the v0 env via `env.set_kwargs(**extra_env_kwargs)` (e.g. `max_total_completion_tokens`, `max_seq_len`, `timeout_seconds`) — typically auto-populated by the orchestrator, distinct from the `args` passed at construction.""" diff --git a/verifiers/v1/configs/cli/validate.py b/verifiers/v1/configs/cli/validate.py index 4197703028..7fad910b17 100644 --- a/verifiers/v1/configs/cli/validate.py +++ b/verifiers/v1/configs/cli/validate.py @@ -3,8 +3,8 @@ from pydantic import AliasChoices, Field, SerializeAsAny, model_validator from pydantic_config import BaseConfig -from verifiers.v1.runtimes import DockerConfig, RuntimeConfig from verifiers.v1.configs.taskset import TasksetConfig +from verifiers.v1.runtimes import DockerConfig, RuntimeConfig class CheckTimeoutConfig(BaseConfig): diff --git a/verifiers/v1/configs/retries.py b/verifiers/v1/configs/retries.py index 85f6e8bc14..488a4e4860 100644 --- a/verifiers/v1/configs/retries.py +++ b/verifiers/v1/configs/retries.py @@ -11,7 +11,7 @@ class RetryConfig(BaseConfig): max_retries: int = Field(0, ge=0) """Whole-rollout retries beyond the first attempt. Off by default — the SDKs already retry transient per-call faults; rerunning a whole trajectory is opt-in.""" - include: list[str] = [] + include: list[str] = Field(default_factory=list) """Only retry errors whose type is listed. Empty = retry anything not excluded.""" - exclude: list[str] = [] + exclude: list[str] = Field(default_factory=list) """Never retry errors whose type is listed (wins over `include`).""" diff --git a/verifiers/v1/configs/task.py b/verifiers/v1/configs/task.py index 889cce1067..13dcb62791 100644 --- a/verifiers/v1/configs/task.py +++ b/verifiers/v1/configs/task.py @@ -2,7 +2,7 @@ from __future__ import annotations -from pydantic import model_validator +from pydantic import Field, model_validator from pydantic_config import BaseConfig from verifiers.v1.configs.judge import Judges, check_judges, resolve_judges @@ -16,7 +16,7 @@ class TaskConfig(BaseConfig): Load-time dataset settings belong on `TasksetConfig` instead. """ - judges: Judges = [] + judges: Judges = Field(default_factory=list) """Judge plugins run after task rewards, set through `--env.taskset.task.judges`.""" @model_validator(mode="before") diff --git a/verifiers/v1/decorators.py b/verifiers/v1/decorators.py index aecc218644..15bce8b35d 100644 --- a/verifiers/v1/decorators.py +++ b/verifiers/v1/decorators.py @@ -2,7 +2,8 @@ import asyncio import inspect -from typing import Any, Callable, TypeVar, overload +from collections.abc import Callable +from typing import Any, TypeVar, overload F = TypeVar("F", bound=Callable[..., Any]) diff --git a/verifiers/v1/dialects/__init__.py b/verifiers/v1/dialects/__init__.py index 4e828061bf..c195b31c72 100644 --- a/verifiers/v1/dialects/__init__.py +++ b/verifiers/v1/dialects/__init__.py @@ -14,11 +14,11 @@ the wire format from the route a request arrived on.""" __all__ = [ - "Dialect", "DIALECTS", "FINISH_REASONS", "AnthropicDialect", "ChatDialect", + "Dialect", "ResponsesDialect", "StreamParser", "parse_message", diff --git a/verifiers/v1/dialects/chat.py b/verifiers/v1/dialects/chat.py index ed9a1b8514..23277f4c4f 100644 --- a/verifiers/v1/dialects/chat.py +++ b/verifiers/v1/dialects/chat.py @@ -8,7 +8,8 @@ import time from collections.abc import Mapping -from dataclasses import dataclass, field as dataclass_field +from dataclasses import dataclass +from dataclasses import field as dataclass_field from typing import Any from openai.types.chat import ChatCompletion diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 0564533e96..0fcee6d171 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -12,31 +12,29 @@ TypeVar, ) - from verifiers.v1.agent import Agent, Agents, _EpisodeAgent +from verifiers.v1.clients import Client, ClientConfig, ModelContext, resolve_client from verifiers.v1.configs.agent import AgentConfig from verifiers.v1.configs.env import ( EnvConfig, _declared_agent_configs, default_agent_harness, ) +from verifiers.v1.episode import Episode +from verifiers.v1.errors import EnvError, boundary from verifiers.v1.harness import Harness, HarnessConfig -from verifiers.v1.clients import Client, ClientConfig, ModelContext, resolve_client from verifiers.v1.interception import ( Interception, make_interception, requires_tunnel, ) +from verifiers.v1.mcp import SharedToolServer, serve_shared from verifiers.v1.retries import run_episode_with_retry from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local -from verifiers.v1.errors import EnvError, boundary from verifiers.v1.task import Task, resolve_server_config -from verifiers.v1.episode import Episode from verifiers.v1.trace import Error, Trace from verifiers.v1.utils.generic import generic_type from verifiers.v1.utils.memory import trim_memory_periodically -from verifiers.v1.mcp import SharedToolServer, serve_shared - logger = logging.getLogger(__name__) @@ -266,7 +264,7 @@ async def run_episode( f"{type(self).__name__}.run() ran no agent — every " "episode must carry at least one run" ) - except Exception as e: + except Exception as e: # noqa: BLE001 - episode boundary records every hook failure # Only the deadline's expiry: inner TimeoutErrors became EnvError already. if isinstance(e, TimeoutError): e = TimeoutError( @@ -280,7 +278,7 @@ async def run_episode( async with asyncio.timeout(self.config.timeout.finalize): async with boundary(EnvError, f"{type(self).__name__}.finalize()"): await self.finalize(task, episode) - except Exception as e: + except Exception as e: # noqa: BLE001 - episode boundary records every hook failure # As above: a TimeoutError here is the deadline's own expiry. if isinstance(e, TimeoutError): e = TimeoutError( diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index 3ad1a66e70..60a8dd6b9a 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -17,7 +17,7 @@ import tomllib from pathlib import Path -from pydantic import field_validator +from pydantic import Field, field_validator import verifiers.v1 as vf from verifiers.v1.types import StrictBaseModel @@ -45,7 +45,7 @@ class Criterion(StrictBaseModel): text: str weight: float = 1.0 """The criterion's share of the reward.""" - choices: list[str] = ["no", "yes"] + choices: list[str] = Field(default_factory=lambda: ["no", "yes"]) """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest evenly spaced by rank. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates.""" @@ -287,7 +287,7 @@ def _check_agents(self) -> None: "(--env.taskset.task.judges), not an agent." ) if isinstance(self.config.solver.runtime, vf.SubprocessConfig): - raise ValueError( + raise TypeError( "agentic-judge plays its judge in the solver's box, but the solver " "(which provisions it) resolves to the subprocess runtime; use " "--env.solver.runtime.type docker or prime" @@ -308,7 +308,7 @@ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: solution, verdict = by_agent["solver"], by_agent["judge"] data = verdict.info.get("verdict") if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list): - raise ValueError( + raise TypeError( f"no verdicts on the judge's trace (expected {VERDICT_FILE} with a " '"verdicts" list)' ) @@ -317,7 +317,7 @@ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: answers: dict[str, str] = {} for entry in data["verdicts"]: if not isinstance(entry, dict): - raise ValueError(f"verdict entry {entry!r} is not an object") + raise TypeError(f"verdict entry {entry!r} is not an object") name = str(entry.get("name")) if name in answers: # Contradictory duplicates must not collapse to whichever came last. diff --git a/verifiers/v1/envs/single_agent/env.py b/verifiers/v1/envs/single_agent/env.py index 401303750b..537e21ad4a 100644 --- a/verifiers/v1/envs/single_agent/env.py +++ b/verifiers/v1/envs/single_agent/env.py @@ -5,8 +5,8 @@ eval is exactly this env (one `agent` seat, one nameless trace per episode). """ -from verifiers.v1.configs.agent import AgentConfig from verifiers.v1.agent import Agents +from verifiers.v1.configs.agent import AgentConfig from verifiers.v1.configs.env import EnvConfig from verifiers.v1.env import Env from verifiers.v1.task import Task diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index 9099ace17f..ccce076402 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -5,9 +5,9 @@ from pydantic import Field +from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData -from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.trace import AgentConfigT, Error, Trace from verifiers.v1.types import StrictBaseModel diff --git a/verifiers/v1/gepa/adapter.py b/verifiers/v1/gepa/adapter.py index 2beb4a2948..d072c52266 100644 --- a/verifiers/v1/gepa/adapter.py +++ b/verifiers/v1/gepa/adapter.py @@ -9,17 +9,17 @@ """ import asyncio -from collections.abc import Awaitable +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass, field -from typing import Any, Callable, Mapping, Sequence +from typing import Any from gepa.core.adapter import EvaluationBatch from pydantic_core import to_jsonable_python from verifiers.v1.clients import ModelContext from verifiers.v1.env import Env -from verifiers.v1.task import Task from verifiers.v1.episode import Episode +from verifiers.v1.task import Task Candidate = dict[str, str] diff --git a/verifiers/v1/gepa/reflection.py b/verifiers/v1/gepa/reflection.py index e606fcaef9..90c6735127 100644 --- a/verifiers/v1/gepa/reflection.py +++ b/verifiers/v1/gepa/reflection.py @@ -5,10 +5,10 @@ """ import asyncio -from typing import Callable +from collections.abc import Callable -from verifiers.v1.gepa.config import GEPAConfig from verifiers.v1.configs.judge import JudgeConfig +from verifiers.v1.gepa.config import GEPAConfig from verifiers.v1.judge import Judge diff --git a/verifiers/v1/gepa/runner.py b/verifiers/v1/gepa/runner.py index 93b4e2e7b6..86a8b9ca90 100644 --- a/verifiers/v1/gepa/runner.py +++ b/verifiers/v1/gepa/runner.py @@ -14,6 +14,7 @@ from verifiers.v1.cli.output import append_episode, output_path, save_config from verifiers.v1.clients import ModelContext, resolve_client from verifiers.v1.env import Env +from verifiers.v1.episode import Episode from verifiers.v1.gepa.adapter import GEPAAdapter from verifiers.v1.gepa.config import GEPAConfig from verifiers.v1.gepa.dataset import ( @@ -21,7 +22,6 @@ split_tasks, ) from verifiers.v1.gepa.reflection import build_reflection_lm -from verifiers.v1.episode import Episode logger = logging.getLogger(__name__) @@ -91,20 +91,20 @@ async def on_complete(episode: Episode) -> None: on_complete=on_complete, reflection_columns=config.reflection_columns, ) - optimize_kwargs: dict = dict( - seed_candidate={"system_prompt": seed_prompt}, - trainset=[task.data.idx for task in train_tasks], - valset=[task.data.idx for task in val_tasks], - adapter=adapter, - reflection_lm=reflection_lm, - max_metric_calls=config.max_total_rollouts, - reflection_minibatch_size=config.reflection_minibatch_size, - run_dir=str(run_dir) if run_dir is not None else None, - seed=config.seed, - display_progress_bar=False, - skip_perfect_score=False, - logger=_GEPALog(), - ) + optimize_kwargs: dict = { + "seed_candidate": {"system_prompt": seed_prompt}, + "trainset": [task.data.idx for task in train_tasks], + "valset": [task.data.idx for task in val_tasks], + "adapter": adapter, + "reflection_lm": reflection_lm, + "max_metric_calls": config.max_total_rollouts, + "reflection_minibatch_size": config.reflection_minibatch_size, + "run_dir": str(run_dir) if run_dir is not None else None, + "seed": config.seed, + "display_progress_bar": False, + "skip_perfect_score": False, + "logger": _GEPALog(), + } return optimize(**optimize_kwargs) finally: loop.run_until_complete(serving.__aexit__(None, None, None)) diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index 8f8eefe691..78fdb3a0a3 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -6,11 +6,10 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar - from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.decorators import discover_decorated, invoke_all from verifiers.v1.errors import HarnessError, boundary -from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.runtimes import ProgramResult, Runtime from verifiers.v1.task import TaskData from verifiers.v1.types import Messages @@ -65,7 +64,7 @@ def resolve_prompt( if system is None or self.APPENDS_SYSTEM_PROMPT: return system if self.APPENDS_SYSTEM_PROMPT else None, prompt if not isinstance(prompt, str): - raise ValueError( + raise TypeError( f"Harness {self.config.id!r} cannot fold a system prompt into a " f"{'Messages' if prompt is not None else 'None'} prompt; set " "APPENDS_SYSTEM_PROMPT to emit it as a system message." diff --git a/verifiers/v1/harnesses/__init__.py b/verifiers/v1/harnesses/__init__.py index ce1b6117a9..fe5166209c 100644 --- a/verifiers/v1/harnesses/__init__.py +++ b/verifiers/v1/harnesses/__init__.py @@ -24,14 +24,14 @@ "CodexHarnessConfig", "KimiCodeHarness", "KimiCodeHarnessConfig", + "MiniSWEAgentHarness", + "MiniSWEAgentHarnessConfig", "NullHarness", "NullHarnessConfig", "PiHarness", "PiHarnessConfig", "PoolHarness", "PoolHarnessConfig", - "MiniSWEAgentHarness", - "MiniSWEAgentHarnessConfig", "RLMHarness", "RLMHarnessConfig", "Terminus2Harness", diff --git a/verifiers/v1/harnesses/bash/harness.py b/verifiers/v1/harnesses/bash/harness.py index 49b0c6efc9..b94989e535 100644 --- a/verifiers/v1/harnesses/bash/harness.py +++ b/verifiers/v1/harnesses/bash/harness.py @@ -2,13 +2,13 @@ import os from pathlib import Path -from verifiers.v1.configs.harness import HarnessConfig -from verifiers.v1.harness import Harness from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.dialects.chat import message_to_wire +from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime -from verifiers.v1.trace import Trace from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text() diff --git a/verifiers/v1/harnesses/bash/program.py b/verifiers/v1/harnesses/bash/program.py index ff17096ad8..aa9326c806 100644 --- a/verifiers/v1/harnesses/bash/program.py +++ b/verifiers/v1/harnesses/bash/program.py @@ -129,17 +129,21 @@ def run_search(query: str, api_key: str, num_results: int = 5) -> str: response.raise_for_status() organic = response.json().get("organic") or [] return format_results(organic[:num_results], query) - except Exception as e: + except Exception as e: # noqa: BLE001 - tool failures are returned to the model return f"search failed ({e}). Try again or rephrase the query." def run_bash(command: str) -> str: try: result = subprocess.run( - ["bash", "-c", command], capture_output=True, text=True, timeout=3600 + ["bash", "-c", command], + capture_output=True, + text=True, + timeout=3600, + check=False, ) return result.stdout + result.stderr - except Exception as e: + except Exception as e: # noqa: BLE001 - tool failures are returned to the model return f"error: {e}" @@ -161,14 +165,14 @@ def run_edit(path: str, old_str: str, new_str: str) -> str: # error as a tool result instead of letting it abort the chat loop. try: content = filepath.read_text() - except Exception as e: + except Exception as e: # noqa: BLE001 - tool failures are returned to the model return f"error: could not read {path}: {e}" count = content.count(old_str) if count != 1: return f"error: old_str must appear exactly once in {path} (found {count})" try: filepath.write_text(content.replace(old_str, new_str, 1)) - except Exception as e: + except Exception as e: # noqa: BLE001 - tool failures are returned to the model return f"error: could not write {path}: {e}" return f"Edited {path}" diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index eb8f9cc8ac..af55a3acc8 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -12,9 +12,9 @@ from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime +from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace from verifiers.v1.types import TextContentPart -from verifiers.v1.task import TaskData logger = logging.getLogger(__name__) @@ -176,7 +176,7 @@ async def resume( ) for part in parts: if not isinstance(part, TextContentPart): - raise ValueError( + raise TypeError( "codex resume supports text user turns only (images go in " "the opening prompt)" ) diff --git a/verifiers/v1/harnesses/mini_swe_agent/harness.py b/verifiers/v1/harnesses/mini_swe_agent/harness.py index 3925e3ae24..b52c96aa6a 100644 --- a/verifiers/v1/harnesses/mini_swe_agent/harness.py +++ b/verifiers/v1/harnesses/mini_swe_agent/harness.py @@ -4,8 +4,8 @@ from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime -from verifiers.v1.trace import Trace from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text() diff --git a/verifiers/v1/harnesses/null/harness.py b/verifiers/v1/harnesses/null/harness.py index 96f85d7d7e..2da3eb8378 100644 --- a/verifiers/v1/harnesses/null/harness.py +++ b/verifiers/v1/harnesses/null/harness.py @@ -1,13 +1,13 @@ import json from pathlib import Path -from verifiers.v1.configs.harness import HarnessConfig -from verifiers.v1.harness import Harness from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.dialects.chat import message_to_wire +from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime -from verifiers.v1.trace import Trace from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text() diff --git a/verifiers/v1/harnesses/pi/harness.py b/verifiers/v1/harnesses/pi/harness.py index 6d1313e6ef..a63ca85349 100644 --- a/verifiers/v1/harnesses/pi/harness.py +++ b/verifiers/v1/harnesses/pi/harness.py @@ -30,10 +30,12 @@ ACP_COMMAND = [ "sh", "-c", - f'export {HOME_VAR}="$HOME"; ' - 'PI_CODING_AGENT_DIR="$PWD/$PI_CODING_AGENT_DIR"; ' - 'export PI_CODING_AGENT_DIR HOME="$PI_CODING_AGENT_DIR"; ' - f'export PATH="{PI_DIR}/node/bin:$PATH"; exec {ACP_BIN}', + ( + f'export {HOME_VAR}="$HOME"; ' + 'PI_CODING_AGENT_DIR="$PWD/$PI_CODING_AGENT_DIR"; ' + 'export PI_CODING_AGENT_DIR HOME="$PI_CODING_AGENT_DIR"; ' + f'export PATH="{PI_DIR}/node/bin:$PATH"; exec {ACP_BIN}' + ), ] INSTALL = r""" diff --git a/verifiers/v1/harnesses/pool/harness.py b/verifiers/v1/harnesses/pool/harness.py index 9e62d723ab..0382bd38d4 100644 --- a/verifiers/v1/harnesses/pool/harness.py +++ b/verifiers/v1/harnesses/pool/harness.py @@ -86,11 +86,13 @@ async def launch( command = [ "sh", "-c", - f'export HOME="$PWD/{pool_home}/home" ' - f'XDG_CONFIG_HOME="$PWD/{pool_home}/config" ' - f'XDG_STATE_HOME="$PWD/{pool_home}/state"; ' - f"exec {POOL_DIR.format(version=self.config.version)}/pool acp " - f"--sandbox disabled --settings {settings}", + ( + f'export HOME="$PWD/{pool_home}/home" ' + f'XDG_CONFIG_HOME="$PWD/{pool_home}/config" ' + f'XDG_STATE_HOME="$PWD/{pool_home}/state"; ' + f"exec {POOL_DIR.format(version=self.config.version)}/pool acp " + f"--sandbox disabled --settings {settings}" + ), ] return await POOL_ACP.run( runtime, diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index 3884dd3562..9bc4194301 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -6,16 +6,16 @@ import shlex from typing import Literal -from pydantic import model_validator +from pydantic import Field, model_validator -from verifiers.v1.configs.harness import HarnessConfig -from verifiers.v1.harness import Harness from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.decorators import metric from verifiers.v1.dialects.chat import message_to_wire +from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime -from verifiers.v1.trace import Trace from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace logger = logging.getLogger(__name__) @@ -35,7 +35,7 @@ class RLMHarnessConfig(HarnessConfig): """Git ref (branch, tag, or commit) of rlm to install.""" max_depth: int = 0 """Recursion depth rlm may spawn sub-harnesses to (RLM_MAX_DEPTH).""" - builtin_skills: list[BuiltinSkill] = [] + builtin_skills: list[BuiltinSkill] = Field(default_factory=list) """Built-in rlm skills to enable (RLM_SKILLS), e.g. `["edit"]`; empty enables none. The tool set is fixed (ipython); the base `skills` field takes SKILL.md paths.""" summarize_at_tokens: int | tuple[int, int] | None = None diff --git a/verifiers/v1/harnesses/terminus_2/harness.py b/verifiers/v1/harnesses/terminus_2/harness.py index e92992e5f4..6e9feb7266 100644 --- a/verifiers/v1/harnesses/terminus_2/harness.py +++ b/verifiers/v1/harnesses/terminus_2/harness.py @@ -5,8 +5,8 @@ from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime -from verifiers.v1.trace import Trace from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text() logger = logging.getLogger(__name__) diff --git a/verifiers/v1/harnesses/terminus_2/program.py b/verifiers/v1/harnesses/terminus_2/program.py index 9eee3cf7b1..916c992b15 100644 --- a/verifiers/v1/harnesses/terminus_2/program.py +++ b/verifiers/v1/harnesses/terminus_2/program.py @@ -28,7 +28,8 @@ async def exec( user: str | int | None = None, ) -> ExecResult: _ = user - result = subprocess.run( + result = await asyncio.to_thread( + subprocess.run, command, shell=True, cwd=cwd, @@ -36,6 +37,7 @@ async def exec( capture_output=True, text=True, timeout=timeout_sec, + check=False, ) return ExecResult( stdout=result.stdout, diff --git a/verifiers/v1/interception/__init__.py b/verifiers/v1/interception/__init__.py index f1c4663825..9d5646f87e 100644 --- a/verifiers/v1/interception/__init__.py +++ b/verifiers/v1/interception/__init__.py @@ -67,16 +67,16 @@ def make_interception( __all__ = [ - "Interception", - "Slot", - "make_interception", - "requires_tunnel", - "InterceptionServer", - "StaticInterceptionPool", - "ElasticInterceptionPool", "BaseInterceptionConfig", + "ElasticInterceptionPool", + "ElasticInterceptionPoolConfig", + "Interception", "InterceptionConfig", + "InterceptionServer", "InterceptionServerConfig", + "Slot", + "StaticInterceptionPool", "StaticInterceptionPoolConfig", - "ElasticInterceptionPoolConfig", + "make_interception", + "requires_tunnel", ] diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 13a7e300f4..b9e5574d19 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -33,9 +33,9 @@ from pydantic import TypeAdapter, ValidationError from pydantic_core import PydanticSerializationError, from_json, to_json +from verifiers.v1 import graph from verifiers.v1.dialects import DIALECTS, Dialect from verifiers.v1.dialects.base import is_sse_done_event -from verifiers.v1 import graph from verifiers.v1.errors import ( OverlongPromptError, ProviderError, @@ -382,7 +382,7 @@ def serve(response: Response) -> web.Response: refused = await session.refused() except RolloutError as e: return self._fail(session, dialect, e) - except Exception as e: + except Exception as e: # noqa: BLE001 - surface any task hook failure return self._fail( session, dialect, @@ -456,7 +456,7 @@ def serve(response: Response) -> web.Response: dialect.error_body(str(e)), status=getattr(e, "status_code", 502), ) - except Exception as e: # surface to the program as an API error + except Exception as e: # noqa: BLE001 - surface as an API error error = e logger.warning( "model call failed: id=%s %s: %s", @@ -515,7 +515,7 @@ async def _stream( refused = await session.refused() except RolloutError as e: return self._fail(session, dialect, e) - except Exception as e: + except Exception as e: # noqa: BLE001 - surface any task hook failure return self._fail( session, dialect, TaskError(f"@stop failed: {type(e).__name__}: {e}") ) @@ -563,7 +563,7 @@ async def _stream( return web.json_response( dialect.error_body(str(e)), status=getattr(e, "status_code", 502) ) - except Exception as e: # surface to the program as an API error + except Exception as e: # noqa: BLE001 - surface as an API error error = e logger.warning("model call failed: id=%s %s", session.trace.id, e) return web.json_response(dialect.error_body(str(e)), status=502) @@ -616,7 +616,7 @@ async def _stream( if on_done is not None and is_sse_done_event(chunk): on_done() feed_event(chunk) - except Exception as e: + except Exception as e: # noqa: BLE001 - defer parser failure parser_error = e # forwarded after the turn is committed, below deferred.append(chunk) @@ -625,7 +625,7 @@ async def _stream( if parser_error is None: try: feed_event(chunk) - except Exception as e: + except Exception as e: # noqa: BLE001 - defer parser failure parser_error = e except ConnectionResetError as e: # The harness went away mid-stream; the provider exchange still happened. @@ -700,7 +700,7 @@ async def handle_aux( return web.json_response( dialect.error_body(str(e)), status=getattr(e, "status_code", 502) ) - except Exception as e: + except Exception as e: # noqa: BLE001 - surface auxiliary relay failures logger.warning("aux call failed: id=%s %s", session.trace.id, e) return web.json_response(dialect.error_body(str(e)), status=502) return web.json_response(result) diff --git a/verifiers/v1/interception/tunnel/__init__.py b/verifiers/v1/interception/tunnel/__init__.py index e60dbc8234..c2043bc39d 100644 --- a/verifiers/v1/interception/tunnel/__init__.py +++ b/verifiers/v1/interception/tunnel/__init__.py @@ -20,12 +20,12 @@ def make_tunnel(config: TunnelConfig) -> Tunnel: __all__ = [ - "Tunnel", "BaseTunnelConfig", - "TunnelConfig", - "PrimeTunnel", - "PrimeTunnelConfig", "CustomTunnel", "CustomTunnelConfig", + "PrimeTunnel", + "PrimeTunnelConfig", + "Tunnel", + "TunnelConfig", "make_tunnel", ] diff --git a/verifiers/v1/judge.py b/verifiers/v1/judge.py index 53d224e408..afc3136ba8 100644 --- a/verifiers/v1/judge.py +++ b/verifiers/v1/judge.py @@ -49,8 +49,8 @@ async def correct(self, trace) -> float: from __future__ import annotations import re -from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Callable, Generic, Literal, cast +from collections.abc import Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, Generic, Literal, cast from pydantic import BaseModel from typing_extensions import TypeVar @@ -62,8 +62,8 @@ async def correct(self, trace) -> float: ) from verifiers.v1.dialects.chat import message_to_wire from verifiers.v1.scoring import parse_judge_choice -from verifiers.v1.utils.generic import generic_type from verifiers.v1.types import Messages, StrictBaseModel, Usage +from verifiers.v1.utils.generic import generic_type if TYPE_CHECKING: from verifiers.v1.task import TaskData @@ -81,7 +81,7 @@ class JudgeResponse(StrictBaseModel, Generic[ParsedT]): JudgeView = Literal["last_reply", "full_trace"] -def judge_question(task: "TaskData", question_field: str) -> str: +def judge_question(task: TaskData, question_field: str) -> str: if not question_field: return task.prompt_text question = getattr(task, question_field, None) @@ -94,7 +94,7 @@ def judge_question(task: "TaskData", question_field: str) -> str: return str(question) -def judge_response(trace: "Trace", view: JudgeView) -> str: +def judge_response(trace: Trace, view: JudgeView) -> str: return trace.transcript if view == "full_trace" else trace.last_reply @@ -147,9 +147,7 @@ def build_messages(self, **fields: Any) -> str | Messages: pattern = re.compile(r"\{(" + "|".join(map(re.escape, fields)) + r")\}") return pattern.sub(lambda m: str(fields[m.group(1)]), template) - async def score( - self, task: "TaskData", trace: "Trace" - ) -> float | Mapping[str, float]: + async def score(self, task: TaskData, trace: Trace) -> float | Mapping[str, float]: raise NotImplementedError( f"{type(self).__name__} implements no `score`, so it can't be plugged via " "`taskset.task.judges`; implement `score` (see verifiers.v1.judges for examples) or " @@ -165,7 +163,7 @@ async def complete( self, messages: str | Messages, *, - trace: "Trace | None" = None, + trace: Trace | None = None, schema: type[BaseModel] | None = None, parse: Callable[[JudgeResponse[Any]], Any] | None = None, **sampling: Any, @@ -216,7 +214,7 @@ async def complete( trace.record_judge(response) async def evaluate( - self, *, trace: "Trace | None" = None, **fields: Any + self, *, trace: Trace | None = None, **fields: Any ) -> JudgeResponse[ParsedT]: messages = self.build_messages(**fields) return await self.complete( diff --git a/verifiers/v1/judges/__init__.py b/verifiers/v1/judges/__init__.py index 36135b2f3e..cd8f624c2f 100644 --- a/verifiers/v1/judges/__init__.py +++ b/verifiers/v1/judges/__init__.py @@ -2,9 +2,9 @@ from verifiers.v1.judges.rubric import Criterion, RubricJudge, RubricJudgeConfig __all__ = [ + "Criterion", "ReferenceJudge", "ReferenceJudgeConfig", - "Criterion", "RubricJudge", "RubricJudgeConfig", ] diff --git a/verifiers/v1/judges/rubric.py b/verifiers/v1/judges/rubric.py index c7a84856c0..0419454846 100644 --- a/verifiers/v1/judges/rubric.py +++ b/verifiers/v1/judges/rubric.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import cast -from pydantic import field_validator +from pydantic import Field, field_validator from verifiers.v1.configs.judge import JudgeConfig from verifiers.v1.judge import Judge, JudgeView, judge_question, judge_response @@ -62,7 +62,7 @@ class Criterion(StrictBaseModel): text: str weight: float = 1.0 """The criterion's share of the reward (overridable per name via `weights` in config).""" - choices: list[str] = ["no", "yes"] + choices: list[str] = Field(default_factory=lambda: ["no", "yes"]) """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest evenly spaced by rank. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates.""" @@ -82,7 +82,7 @@ class RubricJudgeConfig(JudgeConfig): path: Path """A `.toml` or `.json` file containing a `criteria` list. Relative paths resolve against the evaluation's working directory.""" - weights: dict[str, float] = {} + weights: dict[str, float] = Field(default_factory=dict) """Per-criterion weight overrides by criterion name (config wins over the file).""" question_field: str = "" """Task field to fill the prompt's `{question}`; empty = the task's prompt rendered as @@ -187,12 +187,12 @@ def render(c: Criterion) -> str: f"{fence}\n{answer}\n{fence}\n" ) - fields = dict( - question=judge_question(task, self.config.question_field), - response=judge_response(trace, self.config.view), - criteria="\n".join(render(c) for c in batch), - reference=reference, - ) + fields = { + "question": judge_question(task, self.config.question_field), + "response": judge_response(trace, self.config.view), + "criteria": "\n".join(render(c) for c in batch), + "reference": reference, + } if self.config.structured_output: result = await self.evaluate(trace=trace, **fields) verdicts = cast(RubricVerdicts, result.parsed).verdicts diff --git a/verifiers/v1/legacy.py b/verifiers/v1/legacy.py index 32f5866eaf..79228c63b3 100644 --- a/verifiers/v1/legacy.py +++ b/verifiers/v1/legacy.py @@ -22,7 +22,9 @@ import zmq.asyncio from pydantic import ValidationError +from verifiers.v1 import graph from verifiers.v1.clients.config import ClientConfig, TrainClientConfig +from verifiers.v1.episode import Episode from verifiers.v1.serve.server import EnvServer from verifiers.v1.serve.types import ( RunGroupRequest, @@ -31,8 +33,6 @@ RunResponse, ) from verifiers.v1.task import WireTaskData -from verifiers.v1 import graph -from verifiers.v1.episode import Episode from verifiers.v1.trace import ( Error, GenerationSpan, @@ -504,7 +504,6 @@ async def run_legacy_eval(config) -> list[Episode]: import asyncio from verifiers import load_environment - from verifiers.v1.cli.output import append_trace, save_config from verifiers.v1.utils.install import ensure_installed, env_name from verifiers.v1.utils.sampling import sample diff --git a/verifiers/v1/loaders.py b/verifiers/v1/loaders.py index 0049c0c4fb..edfaebf9a1 100644 --- a/verifiers/v1/loaders.py +++ b/verifiers/v1/loaders.py @@ -4,31 +4,30 @@ import importlib import importlib.util import pkgutil +from collections.abc import Callable from types import ModuleType -from typing import Callable from pydantic import ValidationError from pydantic_config import BaseConfig from verifiers.v1.configs.env import EnvConfig +from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.judge import JudgeConfig +from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.env import Env -from verifiers.v1.utils.generic import prefix_validation_error from verifiers.v1.envs.single_agent import SingleAgentEnv -from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.harness import Harness -from verifiers.v1.configs.judge import JudgeConfig from verifiers.v1.judge import Judge, judge_config_cls -from verifiers.v1.utils.install import ensure_installed -from verifiers.v1.utils.generic import generic_type from verifiers.v1.task import Task -from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.taskset import Taskset +from verifiers.v1.utils.generic import generic_type, prefix_validation_error +from verifiers.v1.utils.install import ensure_installed def builtin_harness_ids() -> list[str]: """The harness ids that ship with verifiers (the `verifiers.v1.harnesses` subpackages).""" - import verifiers.v1.harnesses as harnesses + from verifiers.v1 import harnesses return sorted(m.name for m in pkgutil.iter_modules(harnesses.__path__)) @@ -59,7 +58,7 @@ def narrow_plugin_field( if field == "harness" else "" ) - raise ValueError( + raise ValueError( # noqa: TRY004 - Pydantic validators must raise ValueError f"{field}.id needs an id, and none was given (got {ident!r}); " f"pass the id right after the flag{hint}" ) diff --git a/verifiers/v1/mcp/__init__.py b/verifiers/v1/mcp/__init__.py index f7d5db8bb1..5b98f22827 100644 --- a/verifiers/v1/mcp/__init__.py +++ b/verifiers/v1/mcp/__init__.py @@ -1,6 +1,6 @@ from verifiers.v1.mcp.launch import ( - serve, SharedToolServer, + serve, serve_shared, serve_tools, ) @@ -9,11 +9,11 @@ __all__ = [ "ServerBase", - "Toolset", + "SharedToolServer", "SharedToolsetConfig", + "Toolset", "ToolsetConfig", "serve", - "SharedToolServer", "serve_shared", "serve_tools", ] diff --git a/verifiers/v1/mcp/server.py b/verifiers/v1/mcp/server.py index 487e8e0994..5c35c6fcef 100644 --- a/verifiers/v1/mcp/server.py +++ b/verifiers/v1/mcp/server.py @@ -6,7 +6,8 @@ import inspect import logging import os -from typing import TYPE_CHECKING, Callable, ClassVar, Generic, TypeVar +from collections.abc import Callable +from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar from pydantic import TypeAdapter, ValidationError from pydantic_config import BaseConfig @@ -246,7 +247,7 @@ def _serve(self) -> None: # before setup so an expensive setup does not block port discovery. sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((host, int(os.environ.get("MCP_PORT", 0)))) + sock.bind((host, int(os.environ.get("MCP_PORT", "0")))) port_file = os.environ.get("MCP_PORT_FILE") if port_file: Path(port_file).write_text(str(sock.getsockname()[1])) diff --git a/verifiers/v1/push.py b/verifiers/v1/push.py index f16fde4745..d3122fe5f1 100644 --- a/verifiers/v1/push.py +++ b/verifiers/v1/push.py @@ -272,7 +272,7 @@ def post(path: str, body: dict) -> dict: ) resp.raise_for_status() post(f"/evaluations/{eval_id}/finalize", {"metrics": metrics}) - except Exception as e: + except Exception as e: # noqa: BLE001 - push is best-effort across the full upload logger.warning("--push: upload failed (%s: %s); skipping", type(e).__name__, e) return finish(error=f"{type(e).__name__}: {e}") diff --git a/verifiers/v1/retries.py b/verifiers/v1/retries.py index a19d1ad23e..b9f62234b0 100644 --- a/verifiers/v1/retries.py +++ b/verifiers/v1/retries.py @@ -18,7 +18,6 @@ from collections.abc import Awaitable, Callable from typing import TYPE_CHECKING -from verifiers.v1.configs.retries import RetryConfig from tenacity import ( AsyncRetrying, RetryCallState, @@ -28,6 +27,8 @@ wait_exponential_jitter, ) +from verifiers.v1.configs.retries import RetryConfig + if TYPE_CHECKING: from verifiers.v1.episode import Episode from verifiers.v1.trace import Error diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 3df8632349..8db36ccda9 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -25,9 +25,8 @@ from contextlib import AsyncExitStack, asynccontextmanager from verifiers import __version__ -from verifiers.v1.configs.agent import AgentConfig -from verifiers.v1.harness import Harness from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.agent import AgentConfig from verifiers.v1.decorators import discover_decorated, invoke from verifiers.v1.dialects import parse_message from verifiers.v1.errors import ( @@ -37,19 +36,20 @@ ToolsetError, boundary, ) +from verifiers.v1.harness import Harness from verifiers.v1.interception import ( Interception, InterceptionServer, Slot, requires_tunnel, ) -from verifiers.v1.session import RolloutLimits, RolloutSession +from verifiers.v1.mcp import SharedToolServer, serve_tools from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, make_runtime, ) -from verifiers.v1.mcp import SharedToolServer, serve_tools +from verifiers.v1.session import RolloutLimits, RolloutSession from verifiers.v1.state import state_cls from verifiers.v1.task import Task, TaskData from verifiers.v1.trace import AgentInfo, Trace, TraceTask, VersionInfo @@ -86,9 +86,8 @@ async def _serve_interception( shared_tools.values(), ) server = InterceptionServer(requires_tunnel=tunneled) - async with server: - async with server.acquire(session) as slot: - yield slot + async with server, server.acquire(session) as slot: + yield slot class RolloutRun: @@ -289,7 +288,7 @@ async def open(self) -> bool: # Setup and service provisioning are complete. Apply the runtime's # execution policy while preserving the framework routes the agent uses. await runtime.prepare_execution([self._endpoint, *self._urls.values()]) - except Exception as e: + except Exception as e: # noqa: BLE001 - setup boundary records every rollout failure self.fail(e) return False except BaseException: @@ -344,7 +343,7 @@ async def step(self, messages: Messages | None = None) -> bool: else: self.fail(e) return False - except Exception as e: + except Exception as e: # noqa: BLE001 - harness boundary records every rollout failure real = self._session.error if real is not None and isinstance(e, RolloutError): real.__cause__ = e @@ -419,7 +418,7 @@ async def close(self) -> Trace: self._scoring_timeout, ) trace.timing.scoring.end = time.time() - except Exception as e: + except Exception as e: # noqa: BLE001 - finalize boundary records every rollout failure self.fail(e) finally: trace.is_completed = True diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index f84e2b273b..fdfc7303b5 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -52,24 +52,24 @@ def runtime_is_local(config: RuntimeConfig) -> bool: __all__ = [ - "ProgramResult", - "Runtime", - "RuntimeConfig", - "RuntimeInfo", "BaseRuntimeInfo", - "NetworkPolicyConfig", - "make_runtime", - "runtime_is_local", - "SubprocessConfig", - "SubprocessRuntime", - "SubprocessRuntimeInfo", "DockerConfig", "DockerRuntime", "DockerRuntimeInfo", - "PrimeConfig", - "PrimeRuntime", - "PrimeRuntimeInfo", "ModalConfig", "ModalRuntime", "ModalRuntimeInfo", + "NetworkPolicyConfig", + "PrimeConfig", + "PrimeRuntime", + "PrimeRuntimeInfo", + "ProgramResult", + "Runtime", + "RuntimeConfig", + "RuntimeInfo", + "SubprocessConfig", + "SubprocessRuntime", + "SubprocessRuntimeInfo", + "make_runtime", + "runtime_is_local", ] diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index e8d80cb3b9..78d404a64f 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -13,6 +13,7 @@ from pathlib import PurePosixPath from typing import ClassVar, Self +from pydantic import Field from pydantic_config import BaseConfig from verifiers.v1.errors import SandboxError @@ -98,9 +99,9 @@ def cleanup_at_exit() -> None: class NetworkPolicyConfig(BaseConfig): """Shared execution-time policy surface for runtimes that support it.""" - allow: list[str] = ["*"] + allow: list[str] = Field(default_factory=lambda: ["*"]) """Destinations allowed during execution; `*` leaves egress unrestricted.""" - block: list[str] = [] + block: list[str] = Field(default_factory=list) """Destinations denied during execution.""" @property diff --git a/verifiers/v1/runtimes/docker/__init__.py b/verifiers/v1/runtimes/docker/__init__.py index bbeee85409..12d4cb8c52 100644 --- a/verifiers/v1/runtimes/docker/__init__.py +++ b/verifiers/v1/runtimes/docker/__init__.py @@ -388,4 +388,5 @@ def cleanup(self) -> None: stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30, + check=False, ) diff --git a/verifiers/v1/runtimes/docker/egress.py b/verifiers/v1/runtimes/docker/egress.py index e5956f00c1..2323d6ef8d 100644 --- a/verifiers/v1/runtimes/docker/egress.py +++ b/verifiers/v1/runtimes/docker/egress.py @@ -163,7 +163,7 @@ async def _handle( client.receive_data(head) request = client.next_event() if not isinstance(request, h11.Request): - raise ValueError("expected an HTTP request") + raise TypeError("expected an HTTP request") authorization = next( ( value @@ -327,7 +327,7 @@ async def _handle( response_started = True writer.write(chunk) await _drain(writer) - except Exception: + except Exception: # noqa: BLE001 - proxy failures become a generic 502 if not response_started: with contextlib.suppress(Exception): writer.write( diff --git a/verifiers/v1/runtimes/limiters.py b/verifiers/v1/runtimes/limiters.py index 0128900386..d078db677d 100644 --- a/verifiers/v1/runtimes/limiters.py +++ b/verifiers/v1/runtimes/limiters.py @@ -12,6 +12,7 @@ import os import tempfile import time +from typing import Self _LIMITER_DIR = os.path.join(tempfile.gettempdir(), "vf-rate-limiters") @@ -44,7 +45,7 @@ def _reserve(self) -> float: finally: fcntl.flock(f.fileno(), fcntl.LOCK_UN) - async def __aenter__(self) -> "CreationLimiter": + async def __aenter__(self) -> Self: wait = await asyncio.to_thread(self._reserve) if wait > 0: await asyncio.sleep(wait) diff --git a/verifiers/v1/runtimes/modal.py b/verifiers/v1/runtimes/modal.py index 14f5a67b8f..7c65a7331b 100644 --- a/verifiers/v1/runtimes/modal.py +++ b/verifiers/v1/runtimes/modal.py @@ -203,6 +203,6 @@ async def teardown(self) -> None: return try: await sandbox.terminate.aio() - except Exception as e: + except Exception as e: # noqa: BLE001 - provider teardown is best-effort logger.warning("modal: failed to terminate sandbox %s: %s", self.info.id, e) self._sandbox = None diff --git a/verifiers/v1/runtimes/prime.py b/verifiers/v1/runtimes/prime.py index b8de3386b7..f28364e13f 100644 --- a/verifiers/v1/runtimes/prime.py +++ b/verifiers/v1/runtimes/prime.py @@ -16,7 +16,7 @@ from typing import ClassVar, Literal from urllib.parse import urlsplit -from pydantic import model_validator +from pydantic import Field, model_validator from verifiers.v1.errors import SandboxError from verifiers.v1.runtimes.base import ( @@ -49,7 +49,7 @@ class PrimeConfig(NetworkPolicyConfig): """Request guaranteed (vs best-effort) capacity.""" region: str | None = None """Region to provision in (None = provider-chosen).""" - labels: list[str] = [] + labels: list[str] = Field(default_factory=list) """Labels attached to the sandbox.""" # TaskData.resources uses these units; non-default runtime config values take precedence. cpu: float = 1.0 @@ -332,7 +332,7 @@ async def teardown(self) -> None: if self.info.id is not None: # keep info.id available after teardown try: await client.delete(self.info.id) - except Exception as e: + except Exception as e: # noqa: BLE001 - provider teardown is best-effort logger.warning( "prime: failed to delete sandbox %s: %s", self.info.id, e ) diff --git a/verifiers/v1/runtimes/subprocess.py b/verifiers/v1/runtimes/subprocess.py index e4a74deae3..baa35454db 100644 --- a/verifiers/v1/runtimes/subprocess.py +++ b/verifiers/v1/runtimes/subprocess.py @@ -6,7 +6,7 @@ import shutil import signal from pathlib import Path -from typing import Literal +from typing import ClassVar, Literal from pydantic_config import BaseConfig @@ -30,8 +30,8 @@ class SubprocessRuntimeInfo(SubprocessConfig, BaseRuntimeInfo): class SubprocessRuntime(Runtime): # Share prepared script environments across the worker's per-rollout runtimes. - _interpreters: dict[str, str] = {} - _locks: dict[str, asyncio.Lock] = {} + _interpreters: ClassVar[dict[str, str]] = {} + _locks: ClassVar[dict[str, asyncio.Lock]] = {} def __init__(self, config: SubprocessConfig, name: str | None = None) -> None: super().__init__(name) diff --git a/verifiers/v1/scoring.py b/verifiers/v1/scoring.py index 5406a87cd5..f3a9087670 100644 --- a/verifiers/v1/scoring.py +++ b/verifiers/v1/scoring.py @@ -101,7 +101,7 @@ def verify_boxed_math_answer( timeout_seconds=timeout_seconds, ) ) - except (Exception, MathVerifyTimeout): + except (Exception, MathVerifyTimeout): # noqa: BLE001 - malformed answers score zero return 0.0 diff --git a/verifiers/v1/serve/__init__.py b/verifiers/v1/serve/__init__.py index fdfce74e8e..85940ef270 100644 --- a/verifiers/v1/serve/__init__.py +++ b/verifiers/v1/serve/__init__.py @@ -13,17 +13,17 @@ ) __all__ = [ + "EnvClient", "EnvServer", "EnvServerPool", - "serve_env", - "env_config_data", - "EnvClient", "HealthRequest", "HealthResponse", "InfoRequest", "InfoResponse", - "RunRequest", - "RunResponse", "RunGroupRequest", "RunGroupResponse", + "RunRequest", + "RunResponse", + "env_config_data", + "serve_env", ] diff --git a/verifiers/v1/serve/client.py b/verifiers/v1/serve/client.py index 41ff86cc07..385ab35bd1 100644 --- a/verifiers/v1/serve/client.py +++ b/verifiers/v1/serve/client.py @@ -20,6 +20,7 @@ import zmq.asyncio from verifiers.v1.clients.config import ClientConfig +from verifiers.v1.episode import WireEpisode from verifiers.v1.serve.types import ( BaseRequest, BaseResponse, @@ -32,7 +33,6 @@ RunRequest, RunResponse, ) -from verifiers.v1.episode import WireEpisode from verifiers.v1.trace import WireTrace from verifiers.v1.types import SamplingConfig @@ -86,7 +86,7 @@ async def _request( ) try: data = await asyncio.wait_for(future, timeout) - except (asyncio.TimeoutError, asyncio.CancelledError): + except (TimeoutError, asyncio.CancelledError): self._pending.pop(request_id, None) raise if response_type in (HealthResponse, InfoResponse): @@ -119,7 +119,7 @@ async def health(self, timeout: float = 2.0) -> bool: return ( await self._request(HealthRequest(), HealthResponse, timeout=timeout) ).success - except asyncio.TimeoutError: + except TimeoutError: return False async def wait_for_server_startup( diff --git a/verifiers/v1/serve/types.py b/verifiers/v1/serve/types.py index 90981a41b7..8aaa68eff6 100644 --- a/verifiers/v1/serve/types.py +++ b/verifiers/v1/serve/types.py @@ -3,8 +3,8 @@ from pydantic import BaseModel, Field, field_serializer, model_validator from verifiers.v1.clients.config import ClientConfig -from verifiers.v1.task import WireTaskData # noqa: F401 (docstring reference) from verifiers.v1.episode import WireEpisode +from verifiers.v1.task import WireTaskData # noqa: F401 (docstring reference) from verifiers.v1.trace import WireTrace from verifiers.v1.types import SamplingConfig diff --git a/verifiers/v1/task.py b/verifiers/v1/task.py index f3c819741a..9b59119c6d 100644 --- a/verifiers/v1/task.py +++ b/verifiers/v1/task.py @@ -29,13 +29,13 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, ClassVar, Generic -from pydantic import ConfigDict +from pydantic import ConfigDict, Field from pydantic_config import BaseConfig from typing_extensions import TypeVar +from verifiers.v1.configs.task import TaskConfig from verifiers.v1.decorators import discover_decorated, invoke_all from verifiers.v1.errors import TaskError, boundary -from verifiers.v1.configs.task import TaskConfig from verifiers.v1.state import StateT from verifiers.v1.types import Messages, StrictBaseModel, content_text from verifiers.v1.utils.generic import generic_type @@ -114,11 +114,11 @@ class TaskData(StrictBaseModel): system_prompt: str | None = None image: str | None = None workdir: str | None = None - network_allow: list[str] = ["*"] + network_allow: list[str] = Field(default_factory=lambda: ["*"]) """Execution-time destinations requested by this task. `*` leaves the runtime allowlist unchanged; a concrete list replaces a wildcard or combines with existing entries. Prime runtimes accept host-level entries and require `vm=true`.""" - network_block: list[str] = [] + network_block: list[str] = Field(default_factory=list) """Execution-time destinations denied by this task and combined with runtime blocks. Prime runtimes cannot combine an allowlist and a blocklist.""" timeout: TaskTimeout = TaskTimeout() diff --git a/verifiers/v1/tasksets/harbor/taskset.py b/verifiers/v1/tasksets/harbor/taskset.py index e295ed8f64..efbd34da8a 100644 --- a/verifiers/v1/tasksets/harbor/taskset.py +++ b/verifiers/v1/tasksets/harbor/taskset.py @@ -24,11 +24,11 @@ from pydantic import Field +from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.decorators import reward from verifiers.v1.errors import SandboxError from verifiers.v1.runtimes import Runtime from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout -from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.taskset import Taskset from verifiers.v1.types import StrictBaseModel @@ -83,14 +83,14 @@ class HarborData(TaskData): name, and description. The remaining fields mirror Harbor metadata. """ - keywords: list[str] = [] - authors: list[Author] = [] + keywords: list[str] = Field(default_factory=list) + authors: list[Author] = Field(default_factory=list) difficulty: str | None = None category: str | None = None - tags: list[str] = [] + tags: list[str] = Field(default_factory=list) task_dir: str = "" """Host path to the task dir; used to stage tests/ to verify.""" - verifier_env: dict[str, str] = {} + verifier_env: dict[str, str] = Field(default_factory=dict) """Raw [verifier.env] entries (literals or `${VAR}`/`${VAR:-default}` templates). Resolved against the host environment at scoring time, like `harbor run` — so a verifier that needs judge API keys or configuration actually receives them.""" @@ -243,7 +243,7 @@ def resolve_image( return None -def size_to_mb(size: str | int | float) -> float: +def size_to_mb(size: str | float) -> float: """A Harbor size in MB, from either schema: current integer-MB fields or the legacy schema-1.0 size strings ("8G", "512M", "64K").""" if not isinstance(size, str): @@ -275,7 +275,8 @@ def parse_resources(env: dict, multiplier: float = 1.0) -> TaskResources: def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborData: # Harbor is optional, so importing its schema is deferred until a Harbor task loads. - from harbor.models.task.config import NetworkMode, TaskConfig as HarborTaskConfig + from harbor.models.task.config import NetworkMode + from harbor.models.task.config import TaskConfig as HarborTaskConfig config = tomllib.loads((task_dir / "task.toml").read_text()) parsed = HarborTaskConfig.model_validate(config) diff --git a/verifiers/v1/tasksets/lean/__init__.py b/verifiers/v1/tasksets/lean/__init__.py index 1ab0923828..6a3ee74438 100644 --- a/verifiers/v1/tasksets/lean/__init__.py +++ b/verifiers/v1/tasksets/lean/__init__.py @@ -10,9 +10,9 @@ LEAN_PROJECT_PATH, PROOF_FILE_PATH, LeanConfig, - LeanTaskConfig, LeanDatasetConfig, LeanTask, + LeanTaskConfig, LeanTaskset, ) @@ -21,9 +21,9 @@ "LEAN_PROJECT_PATH", "PROOF_FILE_PATH", "LeanConfig", - "LeanTaskConfig", "LeanDatasetConfig", "LeanTask", + "LeanTaskConfig", "LeanTaskset", "build_starter_file", "expected_protected_signature", diff --git a/verifiers/v1/tasksets/lean/taskset.py b/verifiers/v1/tasksets/lean/taskset.py index b62c146327..58ccdee54e 100644 --- a/verifiers/v1/tasksets/lean/taskset.py +++ b/verifiers/v1/tasksets/lean/taskset.py @@ -14,12 +14,12 @@ from pydantic_config import BaseConfig +from verifiers.v1.configs.task import TaskConfig +from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.decorators import reward from verifiers.v1.runtimes import Runtime from verifiers.v1.state import State -from verifiers.v1.configs.task import TaskConfig from verifiers.v1.task import Task, TaskData, TaskResources -from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.taskset import Taskset from verifiers.v1.tasksets.lean.scoring import ( build_starter_file, @@ -222,4 +222,4 @@ def _build_prompt(self, formal_statement: str, header: str) -> str: return block -__all__ = ["LeanTask", "LeanConfig", "LeanTaskset"] +__all__ = ["LeanConfig", "LeanTask", "LeanTaskset"] diff --git a/verifiers/v1/tasksets/openenv/taskset.py b/verifiers/v1/tasksets/openenv/taskset.py index 07f56331a2..ea7d3f5378 100644 --- a/verifiers/v1/tasksets/openenv/taskset.py +++ b/verifiers/v1/tasksets/openenv/taskset.py @@ -11,7 +11,7 @@ from typing import Any, Self import httpx -from pydantic import model_validator +from pydantic import Field, model_validator import verifiers.v1 as vf @@ -32,9 +32,9 @@ class OpenEnvConfig(vf.TasksetConfig): use_docker: bool = False """Use OpenEnv's Docker provider instead of the default UV provider. The engine runs host-side (in the eval process), so this needs Docker on the host.""" - provider_kwargs: dict[str, Any] = {} + provider_kwargs: dict[str, Any] = Field(default_factory=dict) """Extra arguments for OpenEnv's provider (`GenericEnvClient.from_env`).""" - resets: list[dict[str, Any]] = [{}] + resets: list[dict[str, Any]] = Field(default_factory=lambda: [{}]) """One finite task per set of arguments passed to OpenEnv's `reset`.""" @model_validator(mode="after") diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index ae90009e99..235f56f3dc 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -7,19 +7,18 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Annotated, Any, Generic, Literal -from typing_extensions import TypeVar - import numpy as np from pydantic import Field, PrivateAttr from renderers.base import MultiModalData +from typing_extensions import TypeVar if TYPE_CHECKING: from verifiers.v1.judge import JudgeResponse from verifiers.v1 import graph +from verifiers.v1.configs.agent import AgentConfig, WireAgentConfig from verifiers.v1.errors import ProviderError from verifiers.v1.graph import MessageNode -from verifiers.v1.configs.agent import AgentConfig, WireAgentConfig from verifiers.v1.runtimes import RuntimeInfo from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData diff --git a/verifiers/v1/utils/aio.py b/verifiers/v1/utils/aio.py index a62887f509..ab14b290e0 100644 --- a/verifiers/v1/utils/aio.py +++ b/verifiers/v1/utils/aio.py @@ -24,7 +24,7 @@ async def run_shielded(coro: Awaitable[T]) -> T: await asyncio.shield(task) except asyncio.CancelledError as e: cancelled = e - except BaseException: + except BaseException: # noqa: BLE001, S110 - re-raised or chained below pass # `coro` itself failed → task is done; re-raised (or chained) below if cancelled is not None: raise cancelled from (None if task.cancelled() else task.exception()) diff --git a/verifiers/v1/utils/compile.py b/verifiers/v1/utils/compile.py index faf2d38048..2153e283ef 100644 --- a/verifiers/v1/utils/compile.py +++ b/verifiers/v1/utils/compile.py @@ -38,7 +38,7 @@ def resolve_runtime_config( if ( task.data.workdir is not None and workdir_spec is not None - and getattr(config, "workdir") == workdir_spec.default + and config.workdir == workdir_spec.default ): updates["workdir"] = task.data.workdir task_network_policy = "*" not in task.data.network_allow or bool( diff --git a/verifiers/v1/utils/git.py b/verifiers/v1/utils/git.py index 63cc0e3ae0..43222f1c1b 100644 --- a/verifiers/v1/utils/git.py +++ b/verifiers/v1/utils/git.py @@ -97,7 +97,7 @@ async def capture_patch( ) return raw = await runtime.read(capped) - except Exception as exc: # Capture must never fail the rollout. + except Exception as exc: # noqa: BLE001 - capture must never fail the rollout trace.info["patch_error"] = f"{type(exc).__name__}: {exc}" return finally: @@ -105,7 +105,7 @@ async def capture_patch( # on shared-filesystem runtimes; removal is best-effort by design. try: await runtime.run(["rm", "-f", full, capped], env or {}) - except Exception: # Cleanup must never fail the rollout. + except Exception: # noqa: BLE001, S110 - cleanup must never fail the rollout pass if len(raw) > PATCH_CAP_BYTES: raw = raw[:PATCH_CAP_BYTES] diff --git a/verifiers/v1/utils/version.py b/verifiers/v1/utils/version.py index 67d4060cca..f7f15171f5 100644 --- a/verifiers/v1/utils/version.py +++ b/verifiers/v1/utils/version.py @@ -27,6 +27,7 @@ def verifiers_commit() -> str | None: capture_output=True, text=True, timeout=5, + check=False, ) except (OSError, subprocess.TimeoutExpired): return None