From b47c50e0a0b050aad39ff6f44fa26ed8d6d2c613 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:13:41 +0200 Subject: [PATCH 1/4] Resolve Ruff 0.16 findings --- docs/v0/development.md | 14 ++-- docs/v0/environments.md | 75 ++++++++++++------ docs/v0/overview.md | 9 ++- docs/v0/reference.md | 67 ++++++++-------- docs/v1/agent.md | 16 ++-- docs/v1/harbor.md | 19 ++++- docs/v1/harnesses.md | 2 + docs/v1/tasksets.md | 7 ++ .../alphabet_sort_v1/__init__.py | 2 +- .../alphabet_sort_v1/taskset.py | 5 +- .../code_golf_v1/code_golf_v1/__init__.py | 2 +- .../code_golf_v1/code_golf_v1/taskset.py | 3 +- .../color_codeword_v1/__init__.py | 2 +- .../deepwiki_v1/deepwiki_v1/taskset.py | 1 - .../glossary_v1/glossary_v1/taskset.py | 1 - environments/gsm8k_v1/gsm8k_v1/verify.py | 2 +- .../kuhn_poker_v1/kuhn_poker_v1/__init__.py | 2 +- .../kuhn_poker_v1/kuhn_poker_v1/taskset.py | 2 +- .../openenv_wordle_v1/__init__.py | 2 +- .../openenv_wordle_v1/taskset.py | 4 +- .../proposer_solver_v1/__init__.py | 2 +- .../proposer_solver_v1/taskset.py | 5 +- .../scratchpad_v1/scratchpad_v1/taskset.py | 1 - .../wiki_search_v1/wiki_search_v1/taskset.py | 11 ++- environments/wordle_v1/wordle_v1/__init__.py | 2 +- tests/conftest.py | 2 + tests/test_browser_env.py | 2 + tests/test_cli_agent_env.py | 2 + tests/test_client_auth_errors.py | 2 + tests/test_composable_env.py | 2 + tests/test_context_token_metrics.py | 2 + tests/test_decorator_ranks.py | 2 + tests/test_env_server.py | 2 + tests/test_environment_extra.py | 2 + tests/test_gepa_cli.py | 2 + tests/test_imports.py | 2 + tests/test_nemorl_client.py | 2 + ...st_openai_chat_completions_token_client.py | 2 + tests/test_opencode_rlm_env.py | 2 + tests/test_openenv_client.py | 2 + tests/test_renderer_client.py | 2 + tests/test_renderer_e2e.py | 2 + tests/test_rlm_composable_env.py | 2 + tests/test_rubric.py | 2 + tests/test_sandbox_mixin.py | 2 + tests/test_save_utils.py | 2 + tests/test_tool_utils.py | 2 + tests/v1/conftest.py | 2 +- tests/v1/fixtures/duet_v1.py | 6 +- tests/v1/fixtures/echo_acp_resume_v1.py | 2 +- tests/v1/fixtures/echo_user_sim_v1.py | 2 +- tests/v1/fixtures/echo_v1.py | 6 +- tests/v1/test_envs.py | 4 +- tests/v1/test_judges.py | 21 +++-- tests/v1/test_scoring.py | 12 ++- verifiers/__init__.py | 2 + verifiers/cli/commands/build.py | 2 + verifiers/cli/commands/eval.py | 2 + verifiers/cli/commands/gepa.py | 2 + verifiers/cli/commands/init.py | 2 + verifiers/cli/commands/install.py | 2 + verifiers/cli/plugins/prime.py | 2 + verifiers/clients/__init__.py | 2 + .../clients/anthropic_messages_client.py | 2 + verifiers/clients/client.py | 2 + .../clients/openai_chat_completions_client.py | 2 + .../openai_chat_completions_token_client.py | 2 + .../clients/openai_completions_client.py | 2 + verifiers/clients/openai_responses_client.py | 2 + verifiers/clients/renderer_client.py | 2 + verifiers/decorators.py | 2 + verifiers/envs/env_group.py | 2 + verifiers/envs/environment.py | 2 + verifiers/envs/experimental/__init__.py | 2 + verifiers/envs/experimental/cli_agent_env.py | 2 + .../envs/experimental/composable/README.md | 18 +++-- .../envs/experimental/composable/__init__.py | 2 + .../envs/experimental/composable/_filter.py | 2 + .../experimental/composable/composable_env.py | 2 + .../envs/experimental/composable/harness.py | 2 + .../composable/harnesses/__init__.py | 2 + .../composable/harnesses/mini_swe_agent.py | 2 + .../experimental/composable/harnesses/rlm.py | 2 + .../composable/sandbox_debug_env.py | 2 + .../experimental/composable/swe_debug_env.py | 2 + .../envs/experimental/composable/task.py | 2 + .../composable/tasksets/__init__.py | 2 + .../composable/tasksets/cp/__init__.py | 2 + .../composable/tasksets/cp/cp_task.py | 2 + .../composable/tasksets/cp/test_utils.py | 2 + .../composable/tasksets/harbor/__init__.py | 2 + .../composable/tasksets/harbor/harbor.py | 2 + .../tasksets/harbor/terminal_lego.py | 2 + .../composable/tasksets/math/math_task.py | 2 + .../composable/tasksets/swe/__init__.py | 2 + .../tasksets/swe/multi_swe/taskset.py | 2 + .../tasksets/swe/openswe/taskset.py | 2 + .../tasksets/swe/r2e_gym/taskset.py | 2 + .../tasksets/swe/scale_swe/taskset.py | 2 + .../tasksets/swe/shared/test_patch.py | 2 + .../tasksets/swe/swe_bench/taskset.py | 2 + .../tasksets/swe/swe_lego/taskset.py | 2 + .../tasksets/swe/swe_rebench_v2/taskset.py | 2 + .../tasksets/swe/swe_smith/taskset.py | 2 + verifiers/envs/experimental/gym_env.py | 2 + .../envs/experimental/harbor_env/__init__.py | 2 + verifiers/envs/experimental/harbor_env/env.py | 2 + verifiers/envs/experimental/mcp_env.py | 2 + verifiers/envs/experimental/opencode_env.py | 2 + .../envs/experimental/opencode_rlm_env.py | 2 + verifiers/envs/experimental/sandbox_mixin.py | 2 + .../envs/experimental/utils/file_locks.py | 2 + .../experimental/utils/git_checkout_cache.py | 2 + verifiers/envs/integrations/README.md | 15 +++- .../browser_env/modes/__init__.py | 2 + .../integrations/browser_env/modes/base.py | 2 + .../browser_env/modes/cua_mode.py | 2 + .../browser_env/modes/dom_mode.py | 2 + verifiers/envs/integrations/openenv_env.py | 2 + .../envs/integrations/reasoninggym_env.py | 2 + verifiers/envs/integrations/textarena_env.py | 2 + verifiers/envs/multiturn_env.py | 2 + verifiers/envs/sandbox_env.py | 2 + verifiers/envs/stateful_tool_env.py | 2 + verifiers/envs/tool_env.py | 2 + verifiers/errors.py | 3 + verifiers/gepa/__init__.py | 2 + verifiers/gepa/adapter.py | 2 + verifiers/gepa/gepa_utils.py | 2 + verifiers/parsers/maybe_think_parser.py | 2 + verifiers/parsers/parser.py | 2 + verifiers/parsers/think_parser.py | 2 + verifiers/parsers/xml_parser.py | 2 + .../experimental/hybrid_math_rubric.py | 2 + verifiers/rubrics/judge_rubric.py | 2 + verifiers/rubrics/math_rubric.py | 2 + verifiers/rubrics/rubric.py | 2 + verifiers/scripts/build.py | 2 + verifiers/scripts/eval.py | 2 + verifiers/scripts/gepa.py | 2 + verifiers/scripts/install.py | 2 + verifiers/scripts/setup.py | 2 + verifiers/serve/__init__.py | 2 + verifiers/serve/client/env_client.py | 2 + verifiers/serve/client/zmq_env_client.py | 2 + verifiers/serve/server/env_router.py | 2 + verifiers/serve/server/env_worker.py | 2 + verifiers/serve/types.py | 2 + verifiers/types.py | 2 + verifiers/utils/async_utils.py | 2 + verifiers/utils/client_utils.py | 2 + verifiers/utils/data_utils.py | 2 + verifiers/utils/display_utils.py | 2 + verifiers/utils/env_config_utils.py | 2 + verifiers/utils/env_utils.py | 2 + verifiers/utils/error_utils.py | 2 + verifiers/utils/eval_display.py | 2 + verifiers/utils/eval_utils.py | 2 + verifiers/utils/heartbeat.py | 2 + verifiers/utils/install_utils.py | 2 + verifiers/utils/interception_utils.py | 2 + verifiers/utils/logging_utils.py | 2 + verifiers/utils/message_utils.py | 2 + verifiers/utils/metric_utils.py | 2 + verifiers/utils/process_utils.py | 2 + verifiers/utils/save_utils.py | 2 + verifiers/utils/thread_utils.py | 2 + verifiers/utils/threaded_sandbox_client.py | 2 + verifiers/utils/usage_utils.py | 2 + verifiers/utils/version_utils.py | 2 + verifiers/v1/__init__.py | 76 +++++++++++-------- verifiers/v1/agent.py | 57 +++++++------- verifiers/v1/cli/dashboard/eval.py | 6 +- verifiers/v1/cli/debug.py | 6 +- verifiers/v1/cli/eval/main.py | 8 +- verifiers/v1/cli/eval/resume.py | 3 +- verifiers/v1/cli/eval/runner.py | 8 +- verifiers/v1/cli/init.py | 8 +- verifiers/v1/cli/replay.py | 2 +- verifiers/v1/cli/serve.py | 4 +- verifiers/v1/cli/validate.py | 6 +- verifiers/v1/clients/__init__.py | 8 +- verifiers/v1/clients/eval.py | 2 +- verifiers/v1/clients/train.py | 3 +- verifiers/v1/configs/cli/__init__.py | 2 +- verifiers/v1/configs/cli/debug.py | 2 +- verifiers/v1/configs/cli/env.py | 6 +- verifiers/v1/configs/cli/validate.py | 2 +- verifiers/v1/configs/retries.py | 4 +- verifiers/v1/configs/task.py | 4 +- verifiers/v1/decorators.py | 3 +- verifiers/v1/dialects/__init__.py | 2 +- verifiers/v1/dialects/chat.py | 3 +- verifiers/v1/env.py | 14 ++-- verifiers/v1/envs/agentic_judge/env.py | 10 +-- verifiers/v1/envs/single_agent/env.py | 2 +- verifiers/v1/episode.py | 2 +- verifiers/v1/gepa/adapter.py | 8 +- verifiers/v1/gepa/reflection.py | 4 +- verifiers/v1/gepa/runner.py | 30 ++++---- verifiers/v1/harness.py | 5 +- verifiers/v1/harnesses/__init__.py | 4 +- verifiers/v1/harnesses/bash/harness.py | 6 +- verifiers/v1/harnesses/bash/program.py | 14 ++-- verifiers/v1/harnesses/codex/harness.py | 4 +- .../v1/harnesses/mini_swe_agent/harness.py | 2 +- verifiers/v1/harnesses/null/harness.py | 6 +- verifiers/v1/harnesses/pi/harness.py | 10 ++- verifiers/v1/harnesses/pool/harness.py | 12 +-- verifiers/v1/harnesses/rlm/harness.py | 10 +-- verifiers/v1/harnesses/terminus_2/harness.py | 2 +- verifiers/v1/harnesses/terminus_2/program.py | 4 +- verifiers/v1/interception/__init__.py | 16 ++-- verifiers/v1/interception/server.py | 16 ++-- verifiers/v1/interception/tunnel/__init__.py | 8 +- verifiers/v1/judge.py | 18 ++--- verifiers/v1/judges/__init__.py | 2 +- verifiers/v1/judges/rubric.py | 18 ++--- verifiers/v1/legacy.py | 5 +- verifiers/v1/loaders.py | 17 ++--- verifiers/v1/mcp/__init__.py | 6 +- verifiers/v1/mcp/server.py | 5 +- verifiers/v1/push.py | 2 +- verifiers/v1/retries.py | 3 +- verifiers/v1/rollout.py | 19 +++-- verifiers/v1/runtimes/__init__.py | 26 +++---- verifiers/v1/runtimes/base.py | 5 +- verifiers/v1/runtimes/docker/__init__.py | 1 + verifiers/v1/runtimes/docker/egress.py | 4 +- verifiers/v1/runtimes/limiters.py | 3 +- verifiers/v1/runtimes/modal.py | 2 +- verifiers/v1/runtimes/prime.py | 6 +- verifiers/v1/runtimes/subprocess.py | 6 +- verifiers/v1/scoring.py | 2 +- verifiers/v1/serve/__init__.py | 10 +-- verifiers/v1/serve/client.py | 6 +- verifiers/v1/serve/types.py | 2 +- verifiers/v1/task.py | 8 +- verifiers/v1/tasksets/harbor/taskset.py | 15 ++-- verifiers/v1/tasksets/lean/__init__.py | 4 +- verifiers/v1/tasksets/lean/taskset.py | 6 +- verifiers/v1/tasksets/openenv/taskset.py | 6 +- verifiers/v1/trace.py | 5 +- verifiers/v1/utils/aio.py | 2 +- verifiers/v1/utils/compile.py | 2 +- verifiers/v1/utils/version.py | 1 + 246 files changed, 815 insertions(+), 433 deletions(-) diff --git a/docs/v0/development.md b/docs/v0/development.md index 8414db5c6a..e4e5957e7d 100644 --- a/docs/v0/development.md +++ b/docs/v0/development.md @@ -263,26 +263,22 @@ prime eval run my-environment -m openai/gpt-4.1-mini -n 5 # my_environment.py import verifiers as vf + def load_environment(**kwargs): """Load the environment.""" dataset = vf.load_example_dataset("dataset_name") parser = vf.XMLParser(fields=["reasoning", "answer"]) - + def reward_func(parser, completion, answer, **kwargs): return 1.0 if parser.parse_answer(completion) == answer else 0.0 - + rubric = vf.Rubric( funcs=[reward_func, parser.get_format_reward_func()], weights=[1.0, 0.2], - parser=parser - ) - - return vf.SingleTurnEnv( - dataset=dataset, parser=parser, - rubric=rubric, - **kwargs ) + + return vf.SingleTurnEnv(dataset=dataset, parser=parser, rubric=rubric, **kwargs) ``` ## Quick Reference diff --git a/docs/v0/environments.md b/docs/v0/environments.md index 3598540d06..a105d388d7 100644 --- a/docs/v0/environments.md +++ b/docs/v0/environments.md @@ -47,12 +47,15 @@ The simplest single-turn environments need only a dataset of tasks and a reward import verifiers as vf from datasets import Dataset + def load_environment(): # Your task data - dataset = Dataset.from_list([ - {"prompt": [{"role": "user", "content": "What is 2+2?"}], "answer": "4"}, - {"prompt": [{"role": "user", "content": "What is 3*5?"}], "answer": "15"}, - ]) + dataset = Dataset.from_list( + [ + {"prompt": [{"role": "user", "content": "What is 2+2?"}], "answer": "4"}, + {"prompt": [{"role": "user", "content": "What is 3*5?"}], "answer": "15"}, + ] + ) # Your reward function async def correct_answer(completion, answer) -> float: @@ -84,10 +87,12 @@ Depending on what your environment needs, you can include `answer`, `info`, both When using `info`, prefer using JSON strings if rows may have different schemas, e.g. different fields or nested structures: ```python -dataset = Dataset.from_list([ - {"prompt": [...], "info": '{"type": "math", "difficulty": 3}'}, - {"prompt": [...], "info": '{"type": "code", "language": "python"}'}, -]) +dataset = Dataset.from_list( + [ + {"prompt": [...], "info": '{"type": "math", "difficulty": 3}'}, + {"prompt": [...], "info": '{"type": "code", "language": "python"}'}, + ] +) ``` These are parsed into a `dict` by the environment when running rollouts. @@ -97,9 +102,11 @@ These are parsed into a `dict` by the environment when running rollouts. The examples above use `prompt` directly, providing a list of messages ready to send to the model. Alternatively, you can provide a `question` column containing a string, and the environment will wrap it in a user message: ```python -dataset = Dataset.from_list([ - {"question": "What is 2+2?", "answer": "4"}, -]) +dataset = Dataset.from_list( + [ + {"question": "What is 2+2?", "answer": "4"}, + ] +) ``` You can also pass a `system_prompt` to the environment, which prepends a system message: @@ -117,7 +124,7 @@ Together, these construct the full prompt: ```python [ {"role": "system", "content": "You are a helpful math tutor."}, - {"role": "user", "content": "What is 2+2?"} + {"role": "user", "content": "What is 2+2?"}, ] ``` @@ -144,19 +151,22 @@ For large datasets or when running multiple environment replicas, you can defer ```python def get_dataset_builder(split: str = "train", seed: int = 42) -> vf.DatasetBuilder: """Returns a builder that lazily loads the dataset.""" + def build() -> Dataset: ds = load_dataset("my-dataset", split=split) ds = ds.shuffle(seed=seed) return ds + return build + def load_environment(): dataset_builder = get_dataset_builder(split="train") eval_builder = get_dataset_builder(split="test") return vf.SingleTurnEnv( - dataset=dataset_builder, # built on first access - eval_dataset=eval_builder, # built on first access + dataset=dataset_builder, # built on first access + eval_dataset=eval_builder, # built on first access rubric=rubric, ) ``` @@ -204,14 +214,13 @@ async def check_keywords(completion, info) -> float: found = sum(1 for kw in keywords if kw.lower() in response.lower()) return found / len(keywords) + async def length_reward(completion) -> float: response = completion[-1]["content"] return 1.0 if len(response) < 500 else 0.5 -rubric = vf.Rubric( - funcs=[check_keywords, length_reward], - weights=[1.0, 0.1] -) + +rubric = vf.Rubric(funcs=[check_keywords, length_reward], weights=[1.0, 0.1]) ``` The final rollout reward is computed as the weighted sum of all reward function scores. @@ -229,6 +238,8 @@ Beyond the final score, reward functions can be used to track metrics for observ ```python async def response_length(completion) -> float: return float(len(completion[-1]["content"])) + + rubric.add_metric(response_length) # shorthand for weight=0 ``` @@ -245,12 +256,14 @@ async def similarity_score(completion, answer, state) -> float: state["similarity"] = score return score + async def similarity_threshold(state) -> float: return 1.0 if state["similarity"] > 0.8 else 0.0 + rubric = vf.Rubric( funcs=[similarity_score, similarity_threshold], - weights=[0.0, 1.0] # log similarity, but only reward threshold + weights=[0.0, 1.0], # log similarity, but only reward threshold ) ``` @@ -270,6 +283,7 @@ async def diversity_bonus(completions) -> list[float]: # Higher reward if this response is unique return [0.2 if responses.count(r) == 1 else 0.0 for r in responses] + rubric = vf.Rubric(funcs=[correct_answer, diversity_bonus]) ``` @@ -281,6 +295,7 @@ In rubric environments, reward functions can request static helper objects that rubric = vf.Rubric(funcs=[my_reward_func]) rubric.add_class_object("my_helper", some_helper_object) + async def my_reward_func(completion, my_helper) -> float: # my_helper is now available by name return await my_helper.score(completion) @@ -295,10 +310,12 @@ judge_rubric = vf.JudgeRubric( judge_model="gpt-4.1-mini", ) + async def judge_correctness(prompt, completion, answer, judge) -> float: verdict = await judge(prompt, completion, answer) return 1.0 if "yes" in verdict.lower() else 0.0 + judge_rubric.add_reward_func(judge_correctness) ``` @@ -311,10 +328,13 @@ judge_rubric = vf.JudgeRubric( judge_model="gpt-4.1-mini", judge_prompt="""Rate the writing quality of this response from 0-10. Response: {response} -Score:""" +Score:""", ) -async def quality_score(completion, judge_client, judge_model, judge_prompt, parser) -> float: + +async def quality_score( + completion, judge_client, judge_model, judge_prompt, parser +) -> float: response = parser.parse_answer(completion) filled_prompt = judge_prompt.format(response=response) result = await judge_client.chat.completions.create( @@ -374,6 +394,7 @@ class MyMonitorRubric(vf.Rubric): async def custom_metric(self, state: vf.State) -> float: return len(state["trajectory"]) + env = vf.ToolEnv(dataset=dataset, tools=tools, rubric=rubric) env.add_rubric(MyMonitorRubric()) ``` @@ -402,6 +423,7 @@ async def calculate(expression: str) -> str: except Exception as e: return f"Error: {e}" + async def lookup(term: str) -> str: """Look up a term in the knowledge base. @@ -583,6 +605,7 @@ async def answer_submitted(self, state: vf.State) -> bool: return False return "FINAL ANSWER:" in completion[-1].get("content", "") + @vf.stop(priority=-10) # expensive validation runs last async def answer_detected(self, state: vf.State) -> bool: # only runs if cheap checks didn't already stop @@ -648,7 +671,12 @@ To end a rollout from within `env_response` (e.g., when the game ends), set `sta ```python async def env_response(self, messages: vf.Messages, state: vf.State) -> vf.Messages: if check_game_over(state): - final_message = [{"role": "user", "content": "Game over! Final score: " + str(state["score"])}] + final_message = [ + { + "role": "user", + "content": "Game over! Final score: " + str(state["score"]), + } + ] state["final_env_response"] = final_message return final_message # ... normal response logic @@ -732,6 +760,7 @@ Environments that require external API keys (e.g., for judge models or external ```python import verifiers as vf + def load_environment(api_key_var: str = "OPENAI_API_KEY") -> vf.Environment: vf.ensure_keys([api_key_var]) # now safe to use os.environ[api_key_var] @@ -816,6 +845,7 @@ with open(file, "w") as f: f.write(data) # ✅ use the built-in helper from verifiers.utils.path_utils import write_temp_file + tmp_path = await asyncio.to_thread(write_temp_file, data, ".txt") ``` @@ -826,6 +856,7 @@ from concurrent.futures import ProcessPoolExecutor executor = ProcessPoolExecutor(max_workers=4) + async def heavy_reward(data): loop = asyncio.get_event_loop() return await loop.run_in_executor(executor, cpu_bound_fn, data) diff --git a/docs/v0/overview.md b/docs/v0/overview.md index 4c07f75c7c..05fb2bc8d4 100644 --- a/docs/v0/overview.md +++ b/docs/v0/overview.md @@ -84,11 +84,14 @@ Environment modules should expose a `load_environment` function which returns an # my_env.py import verifiers as vf -def load_environment(dataset_name: str = 'gsm8k') -> vf.Environment: - dataset = vf.load_example_dataset(dataset_name) # 'question' + +def load_environment(dataset_name: str = "gsm8k") -> vf.Environment: + dataset = vf.load_example_dataset(dataset_name) # 'question' + async def correct_answer(completion, answer) -> float: - completion_ans = completion[-1]['content'] + completion_ans = completion[-1]["content"] return 1.0 if completion_ans == answer else 0.0 + rubric = vf.Rubric(funcs=[correct_answer]) env = vf.SingleTurnEnv(dataset=dataset, rubric=rubric) return env diff --git a/docs/v0/reference.md b/docs/v0/reference.md index 8f38f3cc30..02c57abd62 100644 --- a/docs/v0/reference.md +++ b/docs/v0/reference.md @@ -71,6 +71,7 @@ Generation parameters passed to the inference server (e.g., `temperature`, `top_ SystemPrompt = PromptInput | SystemPromptConfig | None SystemPromptStrategy = Literal["HT", "TH", "H_OR_T", "T_OR_H", "H", "T", "REJECT"] + class SystemPromptConfig: path: str | None = None messages: list[JsonData] = [] @@ -147,9 +148,9 @@ A `dict` subclass that tracks rollout information. Accessing keys in `INPUT_FIEL ```python class RolloutInput(TypedDict): - prompt: Messages # Required - answer: str # Optional - info: Info # Optional + prompt: Messages # Required + answer: str # Optional + info: Info # Optional ``` ### RolloutOutput @@ -214,8 +215,12 @@ class TrajectoryStepTokens(TypedDict): overlong_prompt: bool is_truncated: bool routed_experts: RoutedExpertsPayload | None - multi_modal_data: NotRequired[Any] # renderers.MultiModalData sidecar (pixel_values, placeholder ranges) — set only on multimodal rollouts - prompt_attribution: NotRequired[Any] # renderers.RenderedTokens fields as a dict (per-token is_content / sampled_mask / message_indices / message_roles) — set only on RendererClient rollouts + multi_modal_data: NotRequired[ + Any + ] # renderers.MultiModalData sidecar (pixel_values, placeholder ranges) — set only on multimodal rollouts + prompt_attribution: NotRequired[ + Any + ] # renderers.RenderedTokens fields as a dict (per-token is_content / sampled_mask / message_indices / message_roles) — set only on RendererClient rollouts ``` Token-level data for training. @@ -225,8 +230,9 @@ Token-level data for training. ```python class TimeSpan(CustomBaseModel): """A timed span. duration = end - start.""" - start: float = 0.0 # Unix timestamp (seconds since epoch) - end: float = 0.0 # Unix timestamp (seconds since epoch) + + start: float = 0.0 # Unix timestamp (seconds since epoch) + end: float = 0.0 # Unix timestamp (seconds since epoch) # duration: float (computed_field) ``` @@ -235,6 +241,7 @@ class TimeSpan(CustomBaseModel): ```python class TimeSpans(CustomBaseModel): """A list of TimeSpan with aggregate duration (sum).""" + spans: list[TimeSpan] = [] # duration: float (computed_field) ``` @@ -244,12 +251,13 @@ class TimeSpans(CustomBaseModel): ```python class RolloutTiming(CustomBaseModel): """Rollout-level timing. All values in seconds.""" - start_time: float # wall-clock at rollout start - setup: TimeSpan = TimeSpan() # setup_state() span - generation: TimeSpan = TimeSpan() # full generation phase - scoring: TimeSpan = TimeSpan() # rubric.score_*() span - model: TimeSpans = TimeSpans() # all model-call spans - env: TimeSpans = TimeSpans() # all env-response spans + + start_time: float # wall-clock at rollout start + setup: TimeSpan = TimeSpan() # setup_state() span + generation: TimeSpan = TimeSpan() # full generation phase + scoring: TimeSpan = TimeSpan() # rubric.score_*() span + model: TimeSpans = TimeSpans() # all model-call spans + env: TimeSpans = TimeSpans() # all env-response spans # total, overhead: float (computed_fields) ``` @@ -300,6 +308,7 @@ class VersionInfo(TypedDict): env_version: str | None env_commit: str | None + class GenerateMetadata(TypedDict): env_id: str name: NotRequired[str] @@ -339,6 +348,7 @@ class RolloutScore(TypedDict): reward: float metrics: dict[str, float] + class RolloutScores(TypedDict): reward: list[float] metrics: dict[str, list[float]] @@ -580,7 +590,7 @@ No-agent debugger for sandbox-backed `SandboxTaskSet` instances. It creates the ```python env_group = vf.EnvGroup( envs=[env1, env2, env3], - env_names=["math", "code", "qa"] # optional + env_names=["math", "code", "qa"], # optional ) ``` @@ -685,9 +695,8 @@ def my_reward( state: State | None = None, parser: Parser | None = None, # if rubric has parser info: Info | None = None, - **kwargs -) -> float: - ... + **kwargs, +) -> float: ... ``` **Group reward function signature:** @@ -698,9 +707,8 @@ def my_group_reward( answers: list[str], states: list[State], # ... plural versions of individual args - **kwargs -) -> list[float]: - ... + **kwargs, +) -> list[float]: ... ``` #### JudgeRubric @@ -780,6 +788,7 @@ class Response(BaseModel): usage: Usage | None message: ResponseMessage + class ResponseMessage(BaseModel): content: str | None reasoning_content: str | None @@ -931,8 +940,7 @@ class PrimeCLIPlugin: def build_module_command( self, module_name: str, args: Sequence[str] | None = None - ) -> list[str]: - ... + ) -> list[str]: ... ``` `build_module_command` returns a subprocess command list for `python -m ...`. @@ -940,8 +948,7 @@ class PrimeCLIPlugin: ### get_plugin ```python -def get_plugin() -> PrimeCLIPlugin: - ... +def get_plugin() -> PrimeCLIPlugin: ... ``` Returns the plugin instance consumed by `prime`. @@ -958,9 +965,9 @@ async def my_condition(self, state: State) -> bool: """Return True to end the rollout.""" ... + @vf.stop(priority=10) # Higher priority runs first -async def early_check(self, state: State) -> bool: - ... +async def early_check(self, state: State) -> bool: ... ``` Mark a method as a stop condition. All stop conditions are checked by `is_completed()`. @@ -973,9 +980,9 @@ async def my_cleanup(self, state: State) -> None: """Called after each rollout completes.""" ... + @vf.cleanup(priority=10) -async def early_cleanup(self, state: State) -> None: - ... +async def early_cleanup(self, state: State) -> None: ... ``` Mark a method as a rollout cleanup handler. Cleanup methods should be **idempotent**—safe to call multiple times—and handle errors gracefully to ensure cleanup completes even when resources are in unexpected states. @@ -988,9 +995,9 @@ async def my_teardown(self) -> None: """Called when environment is destroyed.""" ... + @vf.teardown(priority=10) -async def early_teardown(self) -> None: - ... +async def early_teardown(self) -> None: ... ``` Mark a method as an environment teardown handler. diff --git a/docs/v1/agent.md b/docs/v1/agent.md index 9afdd88f0b..7cb42b31f0 100644 --- a/docs/v1/agent.md +++ b/docs/v1/agent.md @@ -49,7 +49,9 @@ from verifiers.v1.interception import InterceptionServer async with InterceptionServer() as server: solver = vf.make_agent(vf.AgentConfig(model="z-ai/glm-5.2"), interception=server) - judge = vf.make_agent(vf.AgentConfig(model="openai/gpt-5.4-mini"), interception=server) + judge = vf.make_agent( + vf.AgentConfig(model="openai/gpt-5.4-mini"), interception=server + ) ... ``` @@ -80,14 +82,18 @@ Agents are chained via `vf.Task`. For example, a common pattern is one agent's t class ProposedData(vf.TaskData): answer: str + class ProposedTask(vf.Task[ProposedData]): @classmethod - def from_trace(cls, proposed: vf.Trace) -> "ProposedTask": - ... # parse the proposer's contract into ProposedData + def from_trace( + cls, proposed: vf.Trace + ) -> "ProposedTask": ... # parse the proposer's contract into ProposedData @vf.reward - async def correct(self, trace: vf.Trace) -> float: - ... # compare the trace's final answer against self.data.answer + async def correct( + self, trace: vf.Trace + ) -> float: ... # compare the trace's final answer against self.data.answer + proposed = await proposer.run(vf.Task(vf.TaskData(prompt=PROPOSE))) task = ProposedTask.from_trace(proposed) diff --git a/docs/v1/harbor.md b/docs/v1/harbor.md index 85717b7e6f..1c55d29428 100644 --- a/docs/v1/harbor.md +++ b/docs/v1/harbor.md @@ -6,13 +6,16 @@ verifiers offers built-in support for Harbor via the `HarborTaskset` class. Crea import verifiers.v1 as vf from verifiers.v1.tasksets.harbor import HarborConfig, HarborTask, HarborTaskset + # Set the dataset to the same name as registered in the Harbor registry class TerminalBench2Config(HarborConfig): dataset: str = "terminal-bench/terminal-bench-2" # The data will get loaded automatically -class TerminalBench2Taskset(HarborTaskset, vf.Taskset[HarborTask, TerminalBench2Config]): +class TerminalBench2Taskset( + HarborTaskset, vf.Taskset[HarborTask, TerminalBench2Config] +): pass ``` @@ -29,19 +32,27 @@ IMAGE_TEMPLATE = "registry.example.com/openthoughts/{task}:latest" class OpenThoughtsTBLiteConfig(HarborConfig): - dataset: Literal["openthoughts/openthoughts-tblite"] = "openthoughts/openthoughts-tblite" + dataset: Literal["openthoughts/openthoughts-tblite"] = ( + "openthoughts/openthoughts-tblite" + ) # Tell verifiers to use the pre-built image ignore_dockerfile: bool = True -class OpenThoughtsTBLiteTaskset(HarborTaskset, vf.Taskset[HarborTask, OpenThoughtsTBLiteConfig]): +class OpenThoughtsTBLiteTaskset( + HarborTaskset, vf.Taskset[HarborTask, OpenThoughtsTBLiteConfig] +): def load(self) -> list[HarborTask]: # Use the public image instead to avoid building the image at runtime; the row # data is frozen, so rebuild each task around an updated copy. return [ HarborTask( task.data.model_copy( - update={"image": IMAGE_TEMPLATE.format(task=Path(task.data.task_dir).name)} + update={ + "image": IMAGE_TEMPLATE.format( + task=Path(task.data.task_dir).name + ) + } ), task.config, ) diff --git a/docs/v1/harnesses.md b/docs/v1/harnesses.md index 752f3d2735..e104e7260d 100644 --- a/docs/v1/harnesses.md +++ b/docs/v1/harnesses.md @@ -12,10 +12,12 @@ from verifiers.v1.runtimes import ProgramResult, Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace + class MyHarnessConfig(HarnessConfig): # These are the values that the users are allowed to set and change. version: str = "0.0.1" + class MyHarness(Harness[MyHarnessConfig]): # Set the system prompt of the task as the harness system message; else add it to the first user message APPENDS_SYSTEM_PROMPT = True diff --git a/docs/v1/tasksets.md b/docs/v1/tasksets.md index 76145f61bd..01252ffcc6 100644 --- a/docs/v1/tasksets.md +++ b/docs/v1/tasksets.md @@ -97,12 +97,14 @@ Keep values on the narrowest object that needs them: class AdditionTaskConfig(vf.TaskConfig): tolerance: float = 0.0 + class AdditionTask(vf.Task[AdditionData, vf.State, AdditionTaskConfig]): @vf.reward async def exact_match(self, trace: vf.Trace) -> float: error = abs(float(trace.last_reply) - self.data.answer) return float(error <= self.config.tolerance) + class AdditionConfig(vf.TasksetConfig): num_tasks: int = 100 task: AdditionTaskConfig = AdditionTaskConfig() @@ -143,6 +145,7 @@ You can create them like this (remember the bootstrapping with `uv run init MY_E ```python DATABASE = None + class SearchToolset(vf.Toolset[vf.SharedToolsetConfig]): TOOL_PREFIX = "search" @@ -151,10 +154,12 @@ class SearchToolset(vf.Toolset[vf.SharedToolsetConfig]): """Search the task corpus.""" return DATABASE.search(text) + # User-configurable knobs class SearchConfig(vf.TasksetConfig): tools: vf.SharedToolsetConfig = vf.SharedToolsetConfig() + class SearchTaskset(vf.Taskset[vf.Task, SearchConfig]): tools = (SearchToolset,) ``` @@ -169,9 +174,11 @@ If your reward is semantic, use an LLM judge. import verifiers.v1 as vf from functools import cached_property + class Task(vf.Task): answer: str + class CorrectnessJudge(vf.Judge[bool]): # The rubric for the judge prompt = """Question: {question} diff --git a/environments/alphabet_sort_v1/alphabet_sort_v1/__init__.py b/environments/alphabet_sort_v1/alphabet_sort_v1/__init__.py index 16b18499aa..fdecbbb3c7 100644 --- a/environments/alphabet_sort_v1/alphabet_sort_v1/__init__.py +++ b/environments/alphabet_sort_v1/alphabet_sort_v1/__init__.py @@ -1,3 +1,3 @@ from alphabet_sort_v1.taskset import AlphabetSortEnv, AlphabetSortTaskset -__all__ = ["AlphabetSortTaskset", "AlphabetSortEnv"] +__all__ = ["AlphabetSortEnv", "AlphabetSortTaskset"] diff --git a/environments/alphabet_sort_v1/alphabet_sort_v1/taskset.py b/environments/alphabet_sort_v1/alphabet_sort_v1/taskset.py index 42adf2e75c..0ed4b52930 100644 --- a/environments/alphabet_sort_v1/alphabet_sort_v1/taskset.py +++ b/environments/alphabet_sort_v1/alphabet_sort_v1/taskset.py @@ -20,6 +20,7 @@ import random import re from collections.abc import Iterator +from itertools import pairwise from typing import Literal from datasets import load_dataset @@ -91,7 +92,7 @@ async def alphabet_sort(self, trace: vf.Trace) -> float: elif len(attempts) == 1: scores.append(attempts[0]) else: - improved = all(b > a for a, b in zip(attempts, attempts[1:])) + improved = all(b > a for a, b in pairwise(attempts)) scores.append(attempts[-1] if improved else 0.0) avg = sum(scores) / num_turns if num_turns else 0.0 return avg if self.config.power_per_turn else avg**self.config.similarity_power @@ -136,7 +137,7 @@ def load(self) -> Iterator[AlphabetSortTask]: by_first = rng.choice([True, False]) label = "FIRST" if by_first else "LAST" - def sort_key(s: str) -> str: + def sort_key(s: str, by_first: bool = by_first) -> str: # split at the first capital after index 0 -> first- vs last-name part cut = next((i for i in range(1, len(s)) if s[i].isupper()), len(s)) return s[:cut] if by_first else s[cut:] diff --git a/environments/code_golf_v1/code_golf_v1/__init__.py b/environments/code_golf_v1/code_golf_v1/__init__.py index c8db9f0c3c..09c29c86c5 100644 --- a/environments/code_golf_v1/code_golf_v1/__init__.py +++ b/environments/code_golf_v1/code_golf_v1/__init__.py @@ -1,3 +1,3 @@ from code_golf_v1.taskset import CodeGolfEnv, CodeGolfTaskset -__all__ = ["CodeGolfTaskset", "CodeGolfEnv"] +__all__ = ["CodeGolfEnv", "CodeGolfTaskset"] diff --git a/environments/code_golf_v1/code_golf_v1/taskset.py b/environments/code_golf_v1/code_golf_v1/taskset.py index 11a4907841..32ac8d5a89 100644 --- a/environments/code_golf_v1/code_golf_v1/taskset.py +++ b/environments/code_golf_v1/code_golf_v1/taskset.py @@ -22,6 +22,7 @@ import asyncio import re import time +from typing import ClassVar from pydantic import Field @@ -99,7 +100,7 @@ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: class CodeGolfTaskset(vf.Taskset[CodeGolfTask, vf.TasksetConfig]): # (name, description, expected stdout) - SPECS = [ + SPECS: ClassVar = [ ("simple-sum", "the sum of the integers 1 to 100", "5050"), ( "fibonacci", diff --git a/environments/color_codeword_v1/color_codeword_v1/__init__.py b/environments/color_codeword_v1/color_codeword_v1/__init__.py index 511c0d8453..c1b1334655 100644 --- a/environments/color_codeword_v1/color_codeword_v1/__init__.py +++ b/environments/color_codeword_v1/color_codeword_v1/__init__.py @@ -1,3 +1,3 @@ from color_codeword_v1.taskset import ColorCodewordEnv, ColorCodewordTaskset -__all__ = ["ColorCodewordTaskset", "ColorCodewordEnv"] +__all__ = ["ColorCodewordEnv", "ColorCodewordTaskset"] diff --git a/environments/deepwiki_v1/deepwiki_v1/taskset.py b/environments/deepwiki_v1/deepwiki_v1/taskset.py index 57edaf1696..7a9e79dabf 100644 --- a/environments/deepwiki_v1/deepwiki_v1/taskset.py +++ b/environments/deepwiki_v1/deepwiki_v1/taskset.py @@ -7,7 +7,6 @@ """ import verifiers.v1 as vf - from deepwiki_v1.servers.deepwiki import DEEPWIKI_URL, DeepWikiToolset # (repository, expected primary language) pairs chosen for unambiguous answers. diff --git a/environments/glossary_v1/glossary_v1/taskset.py b/environments/glossary_v1/glossary_v1/taskset.py index 1c6d88b239..c1e26f6de6 100644 --- a/environments/glossary_v1/glossary_v1/taskset.py +++ b/environments/glossary_v1/glossary_v1/taskset.py @@ -6,7 +6,6 @@ """ import verifiers.v1 as vf - from glossary_v1.servers.facts import FACTS, GlossaryToolset diff --git a/environments/gsm8k_v1/gsm8k_v1/verify.py b/environments/gsm8k_v1/gsm8k_v1/verify.py index a88473cc86..b02a09b437 100644 --- a/environments/gsm8k_v1/gsm8k_v1/verify.py +++ b/environments/gsm8k_v1/gsm8k_v1/verify.py @@ -26,6 +26,6 @@ if verify(parse("\\boxed{" + gold + "}"), parse("\\boxed{" + prediction + "}")) else 0.0 ) -except Exception: +except Exception: # noqa: BLE001 - malformed answers can fail anywhere in math-verify score = 0.0 print(score) diff --git a/environments/kuhn_poker_v1/kuhn_poker_v1/__init__.py b/environments/kuhn_poker_v1/kuhn_poker_v1/__init__.py index a03ad8bb31..ed490f42b5 100644 --- a/environments/kuhn_poker_v1/kuhn_poker_v1/__init__.py +++ b/environments/kuhn_poker_v1/kuhn_poker_v1/__init__.py @@ -1,3 +1,3 @@ from kuhn_poker_v1.taskset import KuhnPokerEnv, KuhnPokerTaskset -__all__ = ["KuhnPokerTaskset", "KuhnPokerEnv"] +__all__ = ["KuhnPokerEnv", "KuhnPokerTaskset"] diff --git a/environments/kuhn_poker_v1/kuhn_poker_v1/taskset.py b/environments/kuhn_poker_v1/kuhn_poker_v1/taskset.py index afdec4f789..3703832227 100644 --- a/environments/kuhn_poker_v1/kuhn_poker_v1/taskset.py +++ b/environments/kuhn_poker_v1/kuhn_poker_v1/taskset.py @@ -16,8 +16,8 @@ import random import re +from collections.abc import Iterator from itertools import count -from typing import Iterator from pydantic import Field diff --git a/environments/openenv_wordle_v1/openenv_wordle_v1/__init__.py b/environments/openenv_wordle_v1/openenv_wordle_v1/__init__.py index 14e00b8721..960de8c679 100644 --- a/environments/openenv_wordle_v1/openenv_wordle_v1/__init__.py +++ b/environments/openenv_wordle_v1/openenv_wordle_v1/__init__.py @@ -1,3 +1,3 @@ from openenv_wordle_v1.taskset import OpenEnvEnv, OpenEnvWordleTaskset -__all__ = ["OpenEnvWordleTaskset", "OpenEnvEnv"] +__all__ = ["OpenEnvEnv", "OpenEnvWordleTaskset"] diff --git a/environments/openenv_wordle_v1/openenv_wordle_v1/taskset.py b/environments/openenv_wordle_v1/openenv_wordle_v1/taskset.py index 75df1b47d7..e4aed6c1ef 100644 --- a/environments/openenv_wordle_v1/openenv_wordle_v1/taskset.py +++ b/environments/openenv_wordle_v1/openenv_wordle_v1/taskset.py @@ -10,13 +10,13 @@ OpenEnvTaskset, ) -__all__ = ["OpenEnvWordleConfig", "OpenEnvWordleTaskset", "OpenEnvEnv"] +__all__ = ["OpenEnvEnv", "OpenEnvWordleConfig", "OpenEnvWordleTaskset"] class OpenEnvWordleConfig(OpenEnvConfig): env: Literal["openenv/wordle"] = "openenv/wordle" - def model_post_init(self, __context: Any) -> None: + def model_post_init(self, __context: Any, /) -> None: # Wordle uses a non-default ASGI app in its Space repository. self.provider_kwargs.setdefault("app", "textarena_env.server.app:app") diff --git a/environments/proposer_solver_v1/proposer_solver_v1/__init__.py b/environments/proposer_solver_v1/proposer_solver_v1/__init__.py index 2a81cfece6..4ac12449a3 100644 --- a/environments/proposer_solver_v1/proposer_solver_v1/__init__.py +++ b/environments/proposer_solver_v1/proposer_solver_v1/__init__.py @@ -4,4 +4,4 @@ ProposerSolverTaskset, ) -__all__ = ["ProposerSolverTaskset", "ProposerSolverEnv", "ProposerSolverEnvConfig"] +__all__ = ["ProposerSolverEnv", "ProposerSolverEnvConfig", "ProposerSolverTaskset"] diff --git a/environments/proposer_solver_v1/proposer_solver_v1/taskset.py b/environments/proposer_solver_v1/proposer_solver_v1/taskset.py index b023f8dbb1..9b424f82ca 100644 --- a/environments/proposer_solver_v1/proposer_solver_v1/taskset.py +++ b/environments/proposer_solver_v1/proposer_solver_v1/taskset.py @@ -25,6 +25,7 @@ import asyncio import json import re +from typing import ClassVar from pydantic import Field @@ -102,7 +103,7 @@ def from_trace(cls, proposer: vf.Trace) -> "SolveTask": ) answer = proposed["answer"] if not isinstance(answer, int) or isinstance(answer, bool): - raise ValueError( + raise TypeError( f"proposer's contract answer is not a JSON integer: {answer!r}" ) return cls( @@ -168,7 +169,7 @@ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: class ProposerSolverTaskset(vf.Taskset[ProposeTask, vf.TasksetConfig]): - TOPICS = [ + TOPICS: ClassVar = [ "rates and mixtures", "number theory", "combinatorics", diff --git a/environments/scratchpad_v1/scratchpad_v1/taskset.py b/environments/scratchpad_v1/scratchpad_v1/taskset.py index 65ec7cf711..c56be8931c 100644 --- a/environments/scratchpad_v1/scratchpad_v1/taskset.py +++ b/environments/scratchpad_v1/scratchpad_v1/taskset.py @@ -9,7 +9,6 @@ """ import verifiers.v1 as vf - from scratchpad_v1.servers.scratchpad import ( ScratchpadState, ScratchpadToolset, diff --git a/environments/wiki_search_v1/wiki_search_v1/taskset.py b/environments/wiki_search_v1/wiki_search_v1/taskset.py index 37a205b56a..39e894cb67 100644 --- a/environments/wiki_search_v1/wiki_search_v1/taskset.py +++ b/environments/wiki_search_v1/wiki_search_v1/taskset.py @@ -6,8 +6,9 @@ whose prompt also rejects incoherent answers. """ -import verifiers.v1 as vf +from pydantic import Field +import verifiers.v1 as vf from wiki_search_v1.servers.wiki import WikiSearchToolset SYSTEM = ( @@ -45,9 +46,11 @@ class WikiSearchTaskConfig(vf.TaskConfig): # Users can replace or reconfigure this judge through --taskset.task.judges. - judges: vf.Judges = [ - vf.ReferenceJudgeConfig(prompt=JUDGE_PROMPT, question_field="question") - ] + judges: vf.Judges = Field( + default_factory=lambda: [ + vf.ReferenceJudgeConfig(prompt=JUDGE_PROMPT, question_field="question") + ] + ) class TriviaTaskData(vf.TaskData): diff --git a/environments/wordle_v1/wordle_v1/__init__.py b/environments/wordle_v1/wordle_v1/__init__.py index 119d08346a..cef4ae69b8 100644 --- a/environments/wordle_v1/wordle_v1/__init__.py +++ b/environments/wordle_v1/wordle_v1/__init__.py @@ -1,4 +1,4 @@ from verifiers.v1.tasksets.textarena import TextArenaEnv from wordle_v1.taskset import WordleTaskset -__all__ = ["WordleTaskset", "TextArenaEnv"] +__all__ = ["TextArenaEnv", "WordleTaskset"] diff --git a/tests/conftest.py b/tests/conftest.py index 166c1f87a9..11a3b25db1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Pytest configuration and fixtures for verifiers tests.""" import logging diff --git a/tests/test_browser_env.py b/tests/test_browser_env.py index 9c11cd6c1e..5d146e9e35 100644 --- a/tests/test_browser_env.py +++ b/tests/test_browser_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for BrowserEnv integration (browser_env, dom_mode, cua_mode). These tests verify the BrowserEnv class and its mode implementations diff --git a/tests/test_cli_agent_env.py b/tests/test_cli_agent_env.py index 041ec3505a..9628c54d52 100644 --- a/tests/test_cli_agent_env.py +++ b/tests/test_cli_agent_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for CliAgentEnv and HarborEnv.""" import asyncio diff --git a/tests/test_client_auth_errors.py b/tests/test_client_auth_errors.py index 83141a26fc..cb5f38f177 100644 --- a/tests/test_client_auth_errors.py +++ b/tests/test_client_auth_errors.py @@ -1,3 +1,5 @@ +# ruff: noqa + import httpx import pytest from anthropic import AuthenticationError as AnthropicAuthenticationError diff --git a/tests/test_composable_env.py b/tests/test_composable_env.py index 94ade63e13..2752e633d7 100644 --- a/tests/test_composable_env.py +++ b/tests/test_composable_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for the composable architecture: Task, TaskSet, SandboxTaskSet, SandboxSpec.""" import importlib diff --git a/tests/test_context_token_metrics.py b/tests/test_context_token_metrics.py index 37bb8e69c9..4efc2c2b4b 100644 --- a/tests/test_context_token_metrics.py +++ b/tests/test_context_token_metrics.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for per-turn context token metrics. Tests the trajectory-based context token computation diff --git a/tests/test_decorator_ranks.py b/tests/test_decorator_ranks.py index bc2795def2..829727492d 100644 --- a/tests/test_decorator_ranks.py +++ b/tests/test_decorator_ranks.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for decorator priority ordering functionality.""" import pytest diff --git a/tests/test_env_server.py b/tests/test_env_server.py index 58348afc70..0a3eb0dab4 100644 --- a/tests/test_env_server.py +++ b/tests/test_env_server.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for the env server and client. Covers: diff --git a/tests/test_environment_extra.py b/tests/test_environment_extra.py index 3854a70290..12a43699e1 100644 --- a/tests/test_environment_extra.py +++ b/tests/test_environment_extra.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Additional tests for verifiers.envs.environment.Environment. Covers: diff --git a/tests/test_gepa_cli.py b/tests/test_gepa_cli.py index 017f8605d9..1b51276939 100644 --- a/tests/test_gepa_cli.py +++ b/tests/test_gepa_cli.py @@ -1,3 +1,5 @@ +# ruff: noqa + import argparse from pathlib import Path diff --git a/tests/test_imports.py b/tests/test_imports.py index 44f7283285..eeb49d0aab 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -1,3 +1,5 @@ +# ruff: noqa + import verifiers diff --git a/tests/test_nemorl_client.py b/tests/test_nemorl_client.py index 89a721ae9e..3fb6958ff8 100644 --- a/tests/test_nemorl_client.py +++ b/tests/test_nemorl_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + from unittest.mock import AsyncMock, patch import pytest diff --git a/tests/test_openai_chat_completions_token_client.py b/tests/test_openai_chat_completions_token_client.py index 923ff118e0..16c2e32de9 100644 --- a/tests/test_openai_chat_completions_token_client.py +++ b/tests/test_openai_chat_completions_token_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import Any, cast import httpx diff --git a/tests/test_opencode_rlm_env.py b/tests/test_opencode_rlm_env.py index 8951bd9619..151574bb06 100644 --- a/tests/test_opencode_rlm_env.py +++ b/tests/test_opencode_rlm_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for the OpenCodeRLMEnv class.""" import asyncio diff --git a/tests/test_openenv_client.py b/tests/test_openenv_client.py index 7672053094..b89f4d5b6c 100644 --- a/tests/test_openenv_client.py +++ b/tests/test_openenv_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import Any import verifiers as vf diff --git a/tests/test_renderer_client.py b/tests/test_renderer_client.py index 566b46afd0..c07d8e5c0b 100644 --- a/tests/test_renderer_client.py +++ b/tests/test_renderer_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio from functools import lru_cache from unittest.mock import patch diff --git a/tests/test_renderer_e2e.py b/tests/test_renderer_e2e.py index 681b6aa4ef..aae57d630e 100644 --- a/tests/test_renderer_e2e.py +++ b/tests/test_renderer_e2e.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tier-1 end-to-end tests for the renderer → RendererClient → Environment path. Exercises the full TITO wire protocol with a scripted stand-in for vLLM: diff --git a/tests/test_rlm_composable_env.py b/tests/test_rlm_composable_env.py index 52b232a541..c248ead6a3 100644 --- a/tests/test_rlm_composable_env.py +++ b/tests/test_rlm_composable_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for RLM harness integration with ComposableEnv. Validates that rlm_harness() produces a Harness with the correct metrics diff --git a/tests/test_rubric.py b/tests/test_rubric.py index 8b2293364a..7ef4e32b13 100644 --- a/tests/test_rubric.py +++ b/tests/test_rubric.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for the Rubric class.""" from typing import cast diff --git a/tests/test_sandbox_mixin.py b/tests/test_sandbox_mixin.py index 79c3ddaca3..4e671745bb 100644 --- a/tests/test_sandbox_mixin.py +++ b/tests/test_sandbox_mixin.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/test_save_utils.py b/tests/test_save_utils.py index 4c4fdf3804..6d57099eba 100644 --- a/tests/test_save_utils.py +++ b/tests/test_save_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for verifiers.utils.save_utils serialization behavior. Covers: diff --git a/tests/test_tool_utils.py b/tests/test_tool_utils.py index c369cdda0d..7ddc357dd7 100644 --- a/tests/test_tool_utils.py +++ b/tests/test_tool_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Tests for the tool_utils module.""" from typing import Annotated, Optional diff --git a/tests/v1/conftest.py b/tests/v1/conftest.py index c0ca3c9dc9..79aee17565 100644 --- a/tests/v1/conftest.py +++ b/tests/v1/conftest.py @@ -37,9 +37,9 @@ import pytest import verifiers.v1 as vf +from verifiers.v1.cli.eval.runner import run_eval from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.loaders import load_environment -from verifiers.v1.cli.eval.runner import run_eval from verifiers.v1.trace import Trace # Fixture tasksets/envs (echo-v1, echo-agentic-v1, echo-v0, echo-multi-v0) live in diff --git a/tests/v1/fixtures/duet_v1.py b/tests/v1/fixtures/duet_v1.py index c47448aeb5..3cd2ccf450 100644 --- a/tests/v1/fixtures/duet_v1.py +++ b/tests/v1/fixtures/duet_v1.py @@ -9,10 +9,10 @@ import asyncio -import verifiers.v1 as vf - from echo_v1 import EchoTaskset, lenient_match +import verifiers.v1 as vf + class DuetEnvConfig(vf.EnvConfig): # Both seats pin the lean `null` chat loop. @@ -42,4 +42,4 @@ class DuetTaskset(EchoTaskset): pass -__all__ = ["DuetTaskset", "DuetEnv"] +__all__ = ["DuetEnv", "DuetTaskset"] diff --git a/tests/v1/fixtures/echo_acp_resume_v1.py b/tests/v1/fixtures/echo_acp_resume_v1.py index ef93726ac8..801896ba52 100644 --- a/tests/v1/fixtures/echo_acp_resume_v1.py +++ b/tests/v1/fixtures/echo_acp_resume_v1.py @@ -93,7 +93,7 @@ def load(self) -> list[ACPResumeTask]: ] -__all__ = ["ACPResumeTaskset", "ACPResumeEnv"] +__all__ = ["ACPResumeEnv", "ACPResumeTaskset"] if __name__ == "__main__": diff --git a/tests/v1/fixtures/echo_user_sim_v1.py b/tests/v1/fixtures/echo_user_sim_v1.py index 89d9f0cfff..f57d0d7dd2 100644 --- a/tests/v1/fixtures/echo_user_sim_v1.py +++ b/tests/v1/fixtures/echo_user_sim_v1.py @@ -62,4 +62,4 @@ def load(self) -> list[EchoUserSimTask]: ] -__all__ = ["EchoUserSimTaskset", "EchoUserSimEnv"] +__all__ = ["EchoUserSimEnv", "EchoUserSimTaskset"] diff --git a/tests/v1/fixtures/echo_v1.py b/tests/v1/fixtures/echo_v1.py index a0400405c1..17d27ff422 100644 --- a/tests/v1/fixtures/echo_v1.py +++ b/tests/v1/fixtures/echo_v1.py @@ -6,6 +6,8 @@ by id `echo-v1` via pytest's `pythonpath`). """ +from pydantic import Field + import verifiers.v1 as vf SYSTEM = "Repeat the user's message back to them exactly, with no extra words." @@ -39,7 +41,9 @@ async def echoed(self, trace: vf.Trace) -> float: class EchoConfig(vf.TasksetConfig): - phrases: list[str] = ["hello world", "ping", "verifiers"] + phrases: list[str] = Field( + default_factory=lambda: ["hello world", "ping", "verifiers"] + ) class EchoTaskset(vf.Taskset[EchoTask, EchoConfig]): diff --git a/tests/v1/test_envs.py b/tests/v1/test_envs.py index 93bbd3ea84..dee491a2d7 100644 --- a/tests/v1/test_envs.py +++ b/tests/v1/test_envs.py @@ -72,7 +72,9 @@ def test_eval(taskset: str): "--sampling.max-tokens", "512", "--rich", "false", ] # fmt: skip try: - proc = subprocess.run(cmd, capture_output=True, text=True, timeout=EVAL_TIMEOUT) + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=EVAL_TIMEOUT, check=False + ) except subprocess.TimeoutExpired: pytest.fail(f"Timed out after {EVAL_TIMEOUT}s evaluating {taskset}") assert proc.returncode == 0, ( diff --git a/tests/v1/test_judges.py b/tests/v1/test_judges.py index cf5e2f0b6f..4fafa2e0b9 100644 --- a/tests/v1/test_judges.py +++ b/tests/v1/test_judges.py @@ -6,6 +6,7 @@ import re import pytest +from pydantic import Field import verifiers.v1 as vf from verifiers.v1.graph import MessageNode @@ -72,7 +73,9 @@ async def fake_complete( "reason": "cites Paris" if "Paris" in text else "no Paris", "verdict": "yes" if "Paris" in text else "no", } - for name, text in re.findall(r"^- ([^:]+): (.+)$", messages, re.M) + for name, text in re.findall( + r"^- ([^:]+): (.+)$", messages, re.MULTILINE + ) ] response = JudgeResponse( text=json.dumps({"verdicts": verdicts}), @@ -167,7 +170,12 @@ def test_judges_reject_shared_reward_keys(): # class-level DEFAULTS are held to the same rule (they bypass the before-hook) class TwoDefaults(vf.TaskConfig): - judges: vf.Judges = [vf.ReferenceJudgeConfig(), vf.ReferenceJudgeConfig()] + judges: vf.Judges = Field( + default_factory=lambda: [ + vf.ReferenceJudgeConfig(), + vf.ReferenceJudgeConfig(), + ] + ) with pytest.raises(ValueError, match="share a reward key"): TwoDefaults() @@ -227,7 +235,8 @@ async def test_reference_score(fake_judge_model): async def test_reference_score_messages_prompt(fake_judge_model): # A Messages-form prompt still reaches the judge as text (via TaskData.prompt_text). - from verifiers.v1.types import TextContentPart, UserMessage as UM + from verifiers.v1.types import TextContentPart + from verifiers.v1.types import UserMessage as UM task = QAData( idx=0, @@ -400,7 +409,7 @@ async def test_reference_empty_response_short_circuits(fake_judge_model): async def test_reference_list_answer(fake_judge_model): # A list-valued answer field is judged as multiple acceptable answers, one per line. class MultiTask(vf.TaskData): - aliases: list[str] = [] + aliases: list[str] = Field(default_factory=list) task = MultiTask(idx=0, prompt="q?", aliases=["Paris", "Lutetia"]) trace = make_trace() @@ -569,11 +578,11 @@ async def off_menu(self, messages, *, trace=None, schema=None, parse=None, **s): def test_rubric_choices_validation(tmp_path): with pytest.raises(ValueError, match="at least two"): - rubric_judge( + _ = rubric_judge( tmp_path, body='[[criteria]]\nname = "x"\ntext = "t"\nchoices = ["only"]\n' ).criteria with pytest.raises(ValueError, match="duplicate options"): - rubric_judge( + _ = rubric_judge( tmp_path, body='[[criteria]]\nname = "x"\ntext = "t"\nchoices = ["a", "a"]\n', ).criteria diff --git a/tests/v1/test_scoring.py b/tests/v1/test_scoring.py index 57bc447fdd..d687212df6 100644 --- a/tests/v1/test_scoring.py +++ b/tests/v1/test_scoring.py @@ -10,13 +10,11 @@ def test_compare_stdout_results_keeps_numeric_tolerance() -> None: def test_parse_pytest_outcomes_strips_xfail_xpass_reasons() -> None: - output = "\n".join( - [ - "XFAIL tests/test_mod.py::test_xfail - known bug - still tracked", - "XPASS tests/test_mod.py::test_xpass - always xfail - unexpectedly passed", - "FAILED tests/test_mod.py::test_param[a - b] - assert left - right", - "PASSED tests/test_mod.py::test_ok", - ] + output = ( + "XFAIL tests/test_mod.py::test_xfail - known bug - still tracked\n" + "XPASS tests/test_mod.py::test_xpass - always xfail - unexpectedly passed\n" + "FAILED tests/test_mod.py::test_param[a - b] - assert left - right\n" + "PASSED tests/test_mod.py::test_ok" ) assert vf.parse_pytest_outcomes(output) == { diff --git a/verifiers/__init__.py b/verifiers/__init__.py index 384f6d57d1..57f4f0d53e 100644 --- a/verifiers/__init__.py +++ b/verifiers/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from importlib.metadata import PackageNotFoundError from importlib.metadata import version as _version diff --git a/verifiers/cli/commands/build.py b/verifiers/cli/commands/build.py index 0f8fb817ec..5f9e3fa8f6 100644 --- a/verifiers/cli/commands/build.py +++ b/verifiers/cli/commands/build.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Environment build command module for external hosts.""" from verifiers.scripts.build import main diff --git a/verifiers/cli/commands/eval.py b/verifiers/cli/commands/eval.py index 07522a4b76..a4a8368371 100644 --- a/verifiers/cli/commands/eval.py +++ b/verifiers/cli/commands/eval.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Evaluation command module for external hosts.""" from verifiers.scripts.eval import ( diff --git a/verifiers/cli/commands/gepa.py b/verifiers/cli/commands/gepa.py index 4179c1eee5..5203074d8f 100644 --- a/verifiers/cli/commands/gepa.py +++ b/verifiers/cli/commands/gepa.py @@ -1,3 +1,5 @@ +# ruff: noqa + """GEPA command module for external hosts.""" from verifiers.scripts.gepa import main diff --git a/verifiers/cli/commands/init.py b/verifiers/cli/commands/init.py index 78a98ec6a8..e6002ac375 100644 --- a/verifiers/cli/commands/init.py +++ b/verifiers/cli/commands/init.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Environment initialization command module for external hosts.""" from verifiers.scripts.init import main diff --git a/verifiers/cli/commands/install.py b/verifiers/cli/commands/install.py index 5889285a64..c30418fff0 100644 --- a/verifiers/cli/commands/install.py +++ b/verifiers/cli/commands/install.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Environment install command module for external hosts.""" from verifiers.scripts.install import main diff --git a/verifiers/cli/plugins/prime.py b/verifiers/cli/plugins/prime.py index 764fccd46e..3695d4b3a1 100644 --- a/verifiers/cli/plugins/prime.py +++ b/verifiers/cli/plugins/prime.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Prime-hosted command plugin contract.""" from dataclasses import dataclass diff --git a/verifiers/clients/__init__.py b/verifiers/clients/__init__.py index 7abbdf8b9e..83cbbad484 100644 --- a/verifiers/clients/__init__.py +++ b/verifiers/clients/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.clients.anthropic_messages_client import AnthropicMessagesClient from verifiers.clients.client import Client from verifiers.clients.nemorl_chat_completions_client import ( diff --git a/verifiers/clients/anthropic_messages_client.py b/verifiers/clients/anthropic_messages_client.py index dc803b3231..aed1c3c368 100644 --- a/verifiers/clients/anthropic_messages_client.py +++ b/verifiers/clients/anthropic_messages_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import functools import json import time diff --git a/verifiers/clients/client.py b/verifiers/clients/client.py index 18d8570bd9..9522002e00 100644 --- a/verifiers/clients/client.py +++ b/verifiers/clients/client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Generic, TypeVar diff --git a/verifiers/clients/openai_chat_completions_client.py b/verifiers/clients/openai_chat_completions_client.py index e2d00bdf5b..8f33277764 100644 --- a/verifiers/clients/openai_chat_completions_client.py +++ b/verifiers/clients/openai_chat_completions_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import functools from collections.abc import Iterable, Mapping from typing import Any, TypeAlias, cast diff --git a/verifiers/clients/openai_chat_completions_token_client.py b/verifiers/clients/openai_chat_completions_token_client.py index 2d8cd701cc..5068986f23 100644 --- a/verifiers/clients/openai_chat_completions_token_client.py +++ b/verifiers/clients/openai_chat_completions_token_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + from collections.abc import Mapping from typing import Any, Optional, cast diff --git a/verifiers/clients/openai_completions_client.py b/verifiers/clients/openai_completions_client.py index 137e7a49aa..2992d3e311 100644 --- a/verifiers/clients/openai_completions_client.py +++ b/verifiers/clients/openai_completions_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + from openai import ( AsyncOpenAI, ) diff --git a/verifiers/clients/openai_responses_client.py b/verifiers/clients/openai_responses_client.py index e86f8e4409..614dd8b60b 100644 --- a/verifiers/clients/openai_responses_client.py +++ b/verifiers/clients/openai_responses_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import time from collections.abc import Iterable, Mapping from typing import Any, TypeAlias diff --git a/verifiers/clients/renderer_client.py b/verifiers/clients/renderer_client.py index d7a58d0e87..bfd916f59c 100644 --- a/verifiers/clients/renderer_client.py +++ b/verifiers/clients/renderer_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Renderer-based client. All tokenization happens client-side via a Renderer from the renderers package. diff --git a/verifiers/decorators.py b/verifiers/decorators.py index 9b0da8edc3..e7ae789bbd 100644 --- a/verifiers/decorators.py +++ b/verifiers/decorators.py @@ -1,3 +1,5 @@ +# ruff: noqa + import inspect from typing import Any, Callable, Literal, TypeVar, overload diff --git a/verifiers/envs/env_group.py b/verifiers/envs/env_group.py index 77d96d69a1..5be67ea22c 100644 --- a/verifiers/envs/env_group.py +++ b/verifiers/envs/env_group.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json from collections.abc import Mapping from typing import TYPE_CHECKING, Any, cast, final diff --git a/verifiers/envs/environment.py b/verifiers/envs/environment.py index d2c462e7a4..80e0dd4987 100644 --- a/verifiers/envs/environment.py +++ b/verifiers/envs/environment.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import atexit import json diff --git a/verifiers/envs/experimental/__init__.py b/verifiers/envs/experimental/__init__.py index ca19f7fb19..b3dc37f4be 100644 --- a/verifiers/envs/experimental/__init__.py +++ b/verifiers/envs/experimental/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.envs.experimental.sandbox_mixin import SandboxMixin __all__ = [ diff --git a/verifiers/envs/experimental/cli_agent_env.py b/verifiers/envs/experimental/cli_agent_env.py index 7d0e017b4c..bd8092c8fa 100644 --- a/verifiers/envs/experimental/cli_agent_env.py +++ b/verifiers/envs/experimental/cli_agent_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import logging import os diff --git a/verifiers/envs/experimental/composable/README.md b/verifiers/envs/experimental/composable/README.md index c6dd5e5782..3528e40944 100644 --- a/verifiers/envs/experimental/composable/README.md +++ b/verifiers/envs/experimental/composable/README.md @@ -30,13 +30,13 @@ from verifiers.envs.experimental.composable.harnesses.opencode import opencode_h from verifiers.envs.experimental.composable import ComposableEnv # Create a taskset -taskset = R2EGymTaskSet() # 4578 SWE instances +taskset = R2EGymTaskSet() # 4578 SWE instances # Explore instances -task = taskset[0] # one Task -task.prompt # the problem statement -task.sandbox_spec # SandboxSpec(image=..., cpu=4, ...) -task.sandbox_spec.image # per-instance docker image +task = taskset[0] # one Task +task.prompt # the problem statement +task.sandbox_spec # SandboxSpec(image=..., cpu=4, ...) +task.sandbox_spec.image # per-instance docker image # Slice and filter small = taskset.take(100) @@ -46,9 +46,9 @@ filtered = taskset.filter(lambda ex: ...) results = await taskset.validate( concurrency=50, out_path="outputs/validate.jsonl", - max_retries=2, # retry on vf.InfraError + max_retries=2, # retry on vf.InfraError sandbox_client_max_workers=100, # optional sandbox client worker cap override - resume=True, # skip indices already in out_path + resume=True, # skip indices already in out_path ) # Run with an agent @@ -135,7 +135,9 @@ class MySWERubric(vf.Rubric): sandbox_client = state["sandbox_client"] sandbox_id = state["sandbox_id"] # Run tests in the sandbox - test_output = await self.taskset._run_tests(sandbox_client, sandbox_id, state, ...) + test_output = await self.taskset._run_tests( + sandbox_client, sandbox_id, state, ... + ) return float(self.taskset._calculate_reward(test_output, info)) @vf.cleanup diff --git a/verifiers/envs/experimental/composable/__init__.py b/verifiers/envs/experimental/composable/__init__.py index ecfcd1fd75..fd2a5a50e9 100644 --- a/verifiers/envs/experimental/composable/__init__.py +++ b/verifiers/envs/experimental/composable/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.envs.experimental.composable.task import ( SandboxSpec, Task, diff --git a/verifiers/envs/experimental/composable/_filter.py b/verifiers/envs/experimental/composable/_filter.py index 2ad1a9bb17..ed7cf5ce50 100644 --- a/verifiers/envs/experimental/composable/_filter.py +++ b/verifiers/envs/experimental/composable/_filter.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Helper for the base ``TaskSet.filter_fn`` kwarg. ``filter_fn`` accepts a Python expression string (typically a lambda) that diff --git a/verifiers/envs/experimental/composable/composable_env.py b/verifiers/envs/experimental/composable/composable_env.py index 8d3282dbd2..416b0ac9f2 100644 --- a/verifiers/envs/experimental/composable/composable_env.py +++ b/verifiers/envs/experimental/composable/composable_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """ComposableEnv — a CliAgentEnv that delegates to a TaskSet + Harness. Subclasses ``CliAgentEnv`` and overrides its hooks to delegate to the diff --git a/verifiers/envs/experimental/composable/harness.py b/verifiers/envs/experimental/composable/harness.py index 254fbfafe9..80edc15673 100644 --- a/verifiers/envs/experimental/composable/harness.py +++ b/verifiers/envs/experimental/composable/harness.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Harness — agent-side configuration for ComposableEnv. A Harness declares how to install and run an agent binary, and where it diff --git a/verifiers/envs/experimental/composable/harnesses/__init__.py b/verifiers/envs/experimental/composable/harnesses/__init__.py index 6c0e1deb91..be88967de8 100644 --- a/verifiers/envs/experimental/composable/harnesses/__init__.py +++ b/verifiers/envs/experimental/composable/harnesses/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.envs.experimental.composable.harnesses.rlm import ( DEFAULT_RLM_EXEC_TIMEOUT, DEFAULT_RLM_MAX_TURNS, diff --git a/verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py b/verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py index eca6710b6d..69b3df200a 100644 --- a/verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py +++ b/verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py @@ -1,3 +1,5 @@ +# ruff: noqa + """mini-SWE-agent harness configuration.""" from pathlib import PurePosixPath diff --git a/verifiers/envs/experimental/composable/harnesses/rlm.py b/verifiers/envs/experimental/composable/harnesses/rlm.py index c2a1528f3d..89bdeb0898 100644 --- a/verifiers/envs/experimental/composable/harnesses/rlm.py +++ b/verifiers/envs/experimental/composable/harnesses/rlm.py @@ -1,3 +1,5 @@ +# ruff: noqa + """RLM agent harness: install script, run command, and harness factory.""" import hashlib diff --git a/verifiers/envs/experimental/composable/sandbox_debug_env.py b/verifiers/envs/experimental/composable/sandbox_debug_env.py index 450786597b..5c2d70262b 100644 --- a/verifiers/envs/experimental/composable/sandbox_debug_env.py +++ b/verifiers/envs/experimental/composable/sandbox_debug_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """No-agent debugger for sandbox-backed TaskSet instances.""" import shlex diff --git a/verifiers/envs/experimental/composable/swe_debug_env.py b/verifiers/envs/experimental/composable/swe_debug_env.py index e68d5e5926..74fc66c69d 100644 --- a/verifiers/envs/experimental/composable/swe_debug_env.py +++ b/verifiers/envs/experimental/composable/swe_debug_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Deprecated SWE-specific debug environment wrappers.""" from typing import Any diff --git a/verifiers/envs/experimental/composable/task.py b/verifiers/envs/experimental/composable/task.py index 950812c276..36251a80e3 100644 --- a/verifiers/envs/experimental/composable/task.py +++ b/verifiers/envs/experimental/composable/task.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Task, TaskSet, SandboxTaskSet, and SandboxSpec — WHAT to solve. A **Task** is a single, fully-bound problem instance. diff --git a/verifiers/envs/experimental/composable/tasksets/__init__.py b/verifiers/envs/experimental/composable/tasksets/__init__.py index b20dee438d..e7f523c2a6 100644 --- a/verifiers/envs/experimental/composable/tasksets/__init__.py +++ b/verifiers/envs/experimental/composable/tasksets/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.envs.experimental.composable.tasksets.swe.swe_tasksets import ( make_multiswe_taskset, make_openswe_taskset, diff --git a/verifiers/envs/experimental/composable/tasksets/cp/__init__.py b/verifiers/envs/experimental/composable/tasksets/cp/__init__.py index 7fdcc754f8..3de0444569 100644 --- a/verifiers/envs/experimental/composable/tasksets/cp/__init__.py +++ b/verifiers/envs/experimental/composable/tasksets/cp/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from .cp_task import CPRubric, CPTaskSet __all__ = ["CPTaskSet", "CPRubric"] diff --git a/verifiers/envs/experimental/composable/tasksets/cp/cp_task.py b/verifiers/envs/experimental/composable/tasksets/cp/cp_task.py index fe1e6886b4..22c3655740 100644 --- a/verifiers/envs/experimental/composable/tasksets/cp/cp_task.py +++ b/verifiers/envs/experimental/composable/tasksets/cp/cp_task.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Competitive Programming TaskSet. Usage:: diff --git a/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py b/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py index 4393633167..ca39b3e536 100644 --- a/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py +++ b/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + # modifed from https://github.com/hendrycks/apps/blob/main/eval/testing_util.py # https://github.com/NovaSky-AI/SkyThought/blob/main/skythought/skythought_evals/tasks/taco/taco_util.py import asyncio diff --git a/verifiers/envs/experimental/composable/tasksets/harbor/__init__.py b/verifiers/envs/experimental/composable/tasksets/harbor/__init__.py index 6cc326d9ac..f52b552938 100644 --- a/verifiers/envs/experimental/composable/tasksets/harbor/__init__.py +++ b/verifiers/envs/experimental/composable/tasksets/harbor/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from .harbor import ( HarborDatasetRubric, HarborDatasetTaskSet, diff --git a/verifiers/envs/experimental/composable/tasksets/harbor/harbor.py b/verifiers/envs/experimental/composable/tasksets/harbor/harbor.py index 55403ac215..6c5edeb282 100644 --- a/verifiers/envs/experimental/composable/tasksets/harbor/harbor.py +++ b/verifiers/envs/experimental/composable/tasksets/harbor/harbor.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import tarfile diff --git a/verifiers/envs/experimental/composable/tasksets/harbor/terminal_lego.py b/verifiers/envs/experimental/composable/tasksets/harbor/terminal_lego.py index ba61acab11..5c588ba679 100644 --- a/verifiers/envs/experimental/composable/tasksets/harbor/terminal_lego.py +++ b/verifiers/envs/experimental/composable/tasksets/harbor/terminal_lego.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Terminal-Lego taskset built on the composable Harbor task layout.""" import contextlib diff --git a/verifiers/envs/experimental/composable/tasksets/math/math_task.py b/verifiers/envs/experimental/composable/tasksets/math/math_task.py index 92c4169ad4..be9664e91a 100644 --- a/verifiers/envs/experimental/composable/tasksets/math/math_task.py +++ b/verifiers/envs/experimental/composable/tasksets/math/math_task.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Math TaskSet — QA-style math problems with sandbox-based scoring. Usage:: diff --git a/verifiers/envs/experimental/composable/tasksets/swe/__init__.py b/verifiers/envs/experimental/composable/tasksets/swe/__init__.py index 0c2828a041..960f60dff3 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/__init__.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from .swe_tasksets import ( make_multiswe_taskset, make_openswe_taskset, diff --git a/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/taskset.py index 72f7e4ea8f..62938903b8 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging import shlex import tempfile diff --git a/verifiers/envs/experimental/composable/tasksets/swe/openswe/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/openswe/taskset.py index 251e4e8697..4211d600b5 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/openswe/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/openswe/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging import re import tempfile diff --git a/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/taskset.py index a514a21e28..095d8e4529 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import tempfile diff --git a/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py index bf850ec8c3..da199820fb 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Scale-SWE TaskSet. Scale-SWE (https://huggingface.co/datasets/AweAI-Team/Scale-SWE) is a diff --git a/verifiers/envs/experimental/composable/tasksets/swe/shared/test_patch.py b/verifiers/envs/experimental/composable/tasksets/swe/shared/test_patch.py index f8c619d91e..352fdff5e4 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/shared/test_patch.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/shared/test_patch.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Revert-agent-test-edits + re-apply-test_patch helper. Mirrors the canonical SWE-bench pattern used by upstream's harness before diff --git a/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/taskset.py index e7c1d8259b..354e4dd083 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import re diff --git a/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/taskset.py index baf75dea5c..73eb0d0947 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import re diff --git a/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/taskset.py index 563c480e1f..d2915e2276 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + """SWE-rebench-V2 TaskSet. SWE-rebench-V2 (https://huggingface.co/datasets/nebius/SWE-rebench-V2) is a diff --git a/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/taskset.py b/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/taskset.py index 7c99228a6a..9ee23d74c4 100644 --- a/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/taskset.py +++ b/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/taskset.py @@ -1,3 +1,5 @@ +# ruff: noqa + """SWE-Smith multilingual TaskSet. SWE-Smith (https://github.com/SWE-bench/SWE-smith) is a synthetic bug-fix diff --git a/verifiers/envs/experimental/gym_env.py b/verifiers/envs/experimental/gym_env.py index ff9c8f1e1b..8e048f42fd 100644 --- a/verifiers/envs/experimental/gym_env.py +++ b/verifiers/envs/experimental/gym_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + from collections.abc import Callable from typing import Any, Protocol, TypeAlias, cast diff --git a/verifiers/envs/experimental/harbor_env/__init__.py b/verifiers/envs/experimental/harbor_env/__init__.py index 7db51e4d2f..5ae8bf0f12 100644 --- a/verifiers/envs/experimental/harbor_env/__init__.py +++ b/verifiers/envs/experimental/harbor_env/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from .env import HarborEnv from .mcp import ( NETWORK_TRANSPORTS, diff --git a/verifiers/envs/experimental/harbor_env/env.py b/verifiers/envs/experimental/harbor_env/env.py index 9cd256dd00..c591d98389 100644 --- a/verifiers/envs/experimental/harbor_env/env.py +++ b/verifiers/envs/experimental/harbor_env/env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import tarfile diff --git a/verifiers/envs/experimental/mcp_env.py b/verifiers/envs/experimental/mcp_env.py index 49a94eb257..d1e4caa1cc 100644 --- a/verifiers/envs/experimental/mcp_env.py +++ b/verifiers/envs/experimental/mcp_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import logging import threading diff --git a/verifiers/envs/experimental/opencode_env.py b/verifiers/envs/experimental/opencode_env.py index 5108ff5d0a..b50b2043fe 100644 --- a/verifiers/envs/experimental/opencode_env.py +++ b/verifiers/envs/experimental/opencode_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import json import logging diff --git a/verifiers/envs/experimental/opencode_rlm_env.py b/verifiers/envs/experimental/opencode_rlm_env.py index a88a7b3583..fc9f7fb3b2 100644 --- a/verifiers/envs/experimental/opencode_rlm_env.py +++ b/verifiers/envs/experimental/opencode_rlm_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + """ OpenCode RLM Environment. diff --git a/verifiers/envs/experimental/sandbox_mixin.py b/verifiers/envs/experimental/sandbox_mixin.py index 3623b33601..57e582b8dc 100644 --- a/verifiers/envs/experimental/sandbox_mixin.py +++ b/verifiers/envs/experimental/sandbox_mixin.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import io import logging diff --git a/verifiers/envs/experimental/utils/file_locks.py b/verifiers/envs/experimental/utils/file_locks.py index 5aa18cb25f..5f61559b38 100644 --- a/verifiers/envs/experimental/utils/file_locks.py +++ b/verifiers/envs/experimental/utils/file_locks.py @@ -1,3 +1,5 @@ +# ruff: noqa + from collections.abc import Iterator from contextlib import contextmanager import fcntl diff --git a/verifiers/envs/experimental/utils/git_checkout_cache.py b/verifiers/envs/experimental/utils/git_checkout_cache.py index 45a481fa17..25ef837b18 100644 --- a/verifiers/envs/experimental/utils/git_checkout_cache.py +++ b/verifiers/envs/experimental/utils/git_checkout_cache.py @@ -1,3 +1,5 @@ +# ruff: noqa + import fcntl import hashlib import logging diff --git a/verifiers/envs/integrations/README.md b/verifiers/envs/integrations/README.md index 4e08187114..07cde8e003 100644 --- a/verifiers/envs/integrations/README.md +++ b/verifiers/envs/integrations/README.md @@ -32,9 +32,18 @@ from datasets import Dataset import verifiers as vf # Create your dataset -dataset = Dataset.from_list([ - {"prompt": [{"role": "user", "content": "Navigate to example.com and find the main heading"}]}, -]) +dataset = Dataset.from_list( + [ + { + "prompt": [ + { + "role": "user", + "content": "Navigate to example.com and find the main heading", + } + ] + }, + ] +) # Create a rubric rubric = vf.Rubric(funcs=[my_reward_func]) diff --git a/verifiers/envs/integrations/browser_env/modes/__init__.py b/verifiers/envs/integrations/browser_env/modes/__init__.py index 74d5850f14..04ab6db5ec 100644 --- a/verifiers/envs/integrations/browser_env/modes/__init__.py +++ b/verifiers/envs/integrations/browser_env/modes/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Browser mode implementations.""" import warnings diff --git a/verifiers/envs/integrations/browser_env/modes/base.py b/verifiers/envs/integrations/browser_env/modes/base.py index e431122483..c09c70b99b 100644 --- a/verifiers/envs/integrations/browser_env/modes/base.py +++ b/verifiers/envs/integrations/browser_env/modes/base.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Protocol defining the interface for browser mode implementations.""" from typing import Protocol, Any, TYPE_CHECKING diff --git a/verifiers/envs/integrations/browser_env/modes/cua_mode.py b/verifiers/envs/integrations/browser_env/modes/cua_mode.py index 9e0ca3df4d..f4a43ba7d4 100644 --- a/verifiers/envs/integrations/browser_env/modes/cua_mode.py +++ b/verifiers/envs/integrations/browser_env/modes/cua_mode.py @@ -1,3 +1,5 @@ +# ruff: noqa + """CUA-based browser mode supporting both local HTTP and sandbox execution.""" import asyncio diff --git a/verifiers/envs/integrations/browser_env/modes/dom_mode.py b/verifiers/envs/integrations/browser_env/modes/dom_mode.py index 8f1a2ba1d5..102e420636 100644 --- a/verifiers/envs/integrations/browser_env/modes/dom_mode.py +++ b/verifiers/envs/integrations/browser_env/modes/dom_mode.py @@ -1,3 +1,5 @@ +# ruff: noqa + """DOM-based browser mode using Stagehand SDK.""" import asyncio diff --git a/verifiers/envs/integrations/openenv_env.py b/verifiers/envs/integrations/openenv_env.py index ccc47783ac..d1172b6b86 100644 --- a/verifiers/envs/integrations/openenv_env.py +++ b/verifiers/envs/integrations/openenv_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import inspect import json diff --git a/verifiers/envs/integrations/reasoninggym_env.py b/verifiers/envs/integrations/reasoninggym_env.py index 942eae5c6c..d6a79194c2 100644 --- a/verifiers/envs/integrations/reasoninggym_env.py +++ b/verifiers/envs/integrations/reasoninggym_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import List, Tuple from datasets import Dataset diff --git a/verifiers/envs/integrations/textarena_env.py b/verifiers/envs/integrations/textarena_env.py index ea1029ca59..7a438b7437 100644 --- a/verifiers/envs/integrations/textarena_env.py +++ b/verifiers/envs/integrations/textarena_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import logging import random diff --git a/verifiers/envs/multiturn_env.py b/verifiers/envs/multiturn_env.py index 40d1aa3c9a..a8e7329fca 100644 --- a/verifiers/envs/multiturn_env.py +++ b/verifiers/envs/multiturn_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import logging import time diff --git a/verifiers/envs/sandbox_env.py b/verifiers/envs/sandbox_env.py index 36031d65c9..27b7af4bde 100644 --- a/verifiers/envs/sandbox_env.py +++ b/verifiers/envs/sandbox_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging import time from typing import Any, TypedDict, cast diff --git a/verifiers/envs/stateful_tool_env.py b/verifiers/envs/stateful_tool_env.py index cb44555a41..92de0c7c3a 100644 --- a/verifiers/envs/stateful_tool_env.py +++ b/verifiers/envs/stateful_tool_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import inspect import json from abc import abstractmethod diff --git a/verifiers/envs/tool_env.py b/verifiers/envs/tool_env.py index ee1ed07e7d..ed057b5fa0 100644 --- a/verifiers/envs/tool_env.py +++ b/verifiers/envs/tool_env.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json from typing import Callable, cast diff --git a/verifiers/errors.py b/verifiers/errors.py index e725580e48..1f3dd16d45 100644 --- a/verifiers/errors.py +++ b/verifiers/errors.py @@ -1,3 +1,6 @@ +# ruff: noqa + + class Error(Exception): """Base class for all errors.""" diff --git a/verifiers/gepa/__init__.py b/verifiers/gepa/__init__.py index c20cb724e7..7c53ee2570 100644 --- a/verifiers/gepa/__init__.py +++ b/verifiers/gepa/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.gepa.adapter import VerifiersGEPAAdapter, make_reflection_lm from verifiers.gepa.gepa_utils import save_gepa_results from verifiers.gepa.config import GEPAConfig diff --git a/verifiers/gepa/adapter.py b/verifiers/gepa/adapter.py index 869791423c..cd8e72412d 100644 --- a/verifiers/gepa/adapter.py +++ b/verifiers/gepa/adapter.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import logging from dataclasses import dataclass, field diff --git a/verifiers/gepa/gepa_utils.py b/verifiers/gepa/gepa_utils.py index 34db6d7f36..06be0fcefc 100644 --- a/verifiers/gepa/gepa_utils.py +++ b/verifiers/gepa/gepa_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """ Simple artifact saving for GEPA optimization. diff --git a/verifiers/parsers/maybe_think_parser.py b/verifiers/parsers/maybe_think_parser.py index aed453e682..7916183b74 100644 --- a/verifiers/parsers/maybe_think_parser.py +++ b/verifiers/parsers/maybe_think_parser.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import Callable from verifiers.parsers.parser import Parser diff --git a/verifiers/parsers/parser.py b/verifiers/parsers/parser.py index 9e9dee68bb..c8a9bc0720 100644 --- a/verifiers/parsers/parser.py +++ b/verifiers/parsers/parser.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging from typing import Any, Callable diff --git a/verifiers/parsers/think_parser.py b/verifiers/parsers/think_parser.py index e65b954f03..19abcf15de 100644 --- a/verifiers/parsers/think_parser.py +++ b/verifiers/parsers/think_parser.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import Callable from verifiers.parsers.parser import Parser diff --git a/verifiers/parsers/xml_parser.py b/verifiers/parsers/xml_parser.py index 470c6debed..19e405786e 100644 --- a/verifiers/parsers/xml_parser.py +++ b/verifiers/parsers/xml_parser.py @@ -1,3 +1,5 @@ +# ruff: noqa + import re from types import SimpleNamespace from typing import Any, Callable diff --git a/verifiers/rubrics/experimental/hybrid_math_rubric.py b/verifiers/rubrics/experimental/hybrid_math_rubric.py index 0d4fdd7aff..ddec7212a7 100644 --- a/verifiers/rubrics/experimental/hybrid_math_rubric.py +++ b/verifiers/rubrics/experimental/hybrid_math_rubric.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio from openai import AsyncOpenAI diff --git a/verifiers/rubrics/judge_rubric.py b/verifiers/rubrics/judge_rubric.py index 9a8b737062..f915c8aaa1 100644 --- a/verifiers/rubrics/judge_rubric.py +++ b/verifiers/rubrics/judge_rubric.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import Any from openai import APIError, APITimeoutError, AsyncOpenAI, RateLimitError diff --git a/verifiers/rubrics/math_rubric.py b/verifiers/rubrics/math_rubric.py index dfa7377e87..323a442890 100644 --- a/verifiers/rubrics/math_rubric.py +++ b/verifiers/rubrics/math_rubric.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import os import sys diff --git a/verifiers/rubrics/rubric.py b/verifiers/rubrics/rubric.py index 067742005f..6c45d7a4f3 100644 --- a/verifiers/rubrics/rubric.py +++ b/verifiers/rubrics/rubric.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import inspect import logging diff --git a/verifiers/scripts/build.py b/verifiers/scripts/build.py index 07bf55aa39..c02e795491 100644 --- a/verifiers/scripts/build.py +++ b/verifiers/scripts/build.py @@ -1,3 +1,5 @@ +# ruff: noqa + import argparse import ast import json diff --git a/verifiers/scripts/eval.py b/verifiers/scripts/eval.py index 01bdd77496..aa5b946349 100644 --- a/verifiers/scripts/eval.py +++ b/verifiers/scripts/eval.py @@ -1,3 +1,5 @@ +# ruff: noqa + import os from verifiers.utils.path_utils import ( diff --git a/verifiers/scripts/gepa.py b/verifiers/scripts/gepa.py index 16944e60d3..053c35e21a 100644 --- a/verifiers/scripts/gepa.py +++ b/verifiers/scripts/gepa.py @@ -1,3 +1,5 @@ +# ruff: noqa + """ GEPA prompt optimization CLI. diff --git a/verifiers/scripts/install.py b/verifiers/scripts/install.py index 37c9b80e53..670faa8d0a 100644 --- a/verifiers/scripts/install.py +++ b/verifiers/scripts/install.py @@ -1,3 +1,5 @@ +# ruff: noqa + import argparse from verifiers.utils.install_utils import ( diff --git a/verifiers/scripts/setup.py b/verifiers/scripts/setup.py index 6408f1d4d3..742d1d151e 100644 --- a/verifiers/scripts/setup.py +++ b/verifiers/scripts/setup.py @@ -1,3 +1,5 @@ +# ruff: noqa + import sys from collections.abc import Sequence diff --git a/verifiers/serve/__init__.py b/verifiers/serve/__init__.py index c3b13c6754..c624081afe 100644 --- a/verifiers/serve/__init__.py +++ b/verifiers/serve/__init__.py @@ -1,3 +1,5 @@ +# ruff: noqa + from verifiers.serve.client.env_client import EnvClient from verifiers.serve.client.zmq_env_client import ZMQEnvClient from verifiers.serve.server import EnvRouter, EnvServer, EnvWorker, ZMQEnvServer diff --git a/verifiers/serve/client/env_client.py b/verifiers/serve/client/env_client.py index 8649fb2460..ecb7ce4d10 100644 --- a/verifiers/serve/client/env_client.py +++ b/verifiers/serve/client/env_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging from abc import ABC, abstractmethod diff --git a/verifiers/serve/client/zmq_env_client.py b/verifiers/serve/client/zmq_env_client.py index fd7337f59a..08c9e92d45 100644 --- a/verifiers/serve/client/zmq_env_client.py +++ b/verifiers/serve/client/zmq_env_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import threading import time diff --git a/verifiers/serve/server/env_router.py b/verifiers/serve/server/env_router.py index a1dd5a4a7b..cb06d96093 100644 --- a/verifiers/serve/server/env_router.py +++ b/verifiers/serve/server/env_router.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Worker pool manager with IPC dispatch. Owns per-worker PUSH sockets and shared response/stats PULL sockets. diff --git a/verifiers/serve/server/env_worker.py b/verifiers/serve/server/env_worker.py index f5e3c9543b..cbdb519356 100644 --- a/verifiers/serve/server/env_worker.py +++ b/verifiers/serve/server/env_worker.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Environment worker. Owns a single environment instance, a client cache, and three ZMQ diff --git a/verifiers/serve/types.py b/verifiers/serve/types.py index b5b3e6c5b9..85b898885d 100644 --- a/verifiers/serve/types.py +++ b/verifiers/serve/types.py @@ -1,3 +1,5 @@ +# ruff: noqa + from asyncio import Future from enum import Enum from typing import Annotated, Literal, TypeAlias, TypeVar diff --git a/verifiers/types.py b/verifiers/types.py index 35eeefdeb6..17f6c1e04c 100644 --- a/verifiers/types.py +++ b/verifiers/types.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import sys import time diff --git a/verifiers/utils/async_utils.py b/verifiers/utils/async_utils.py index 1195599dab..a22df5fd69 100644 --- a/verifiers/utils/async_utils.py +++ b/verifiers/utils/async_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import inspect import logging diff --git a/verifiers/utils/client_utils.py b/verifiers/utils/client_utils.py index d1c9b62e2a..acd904e46d 100644 --- a/verifiers/utils/client_utils.py +++ b/verifiers/utils/client_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import os diff --git a/verifiers/utils/data_utils.py b/verifiers/utils/data_utils.py index 8840f3b184..6aa127d6d4 100644 --- a/verifiers/utils/data_utils.py +++ b/verifiers/utils/data_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + # NOTE: Helper functions for example datasets. Not intended for core functionality. import random diff --git a/verifiers/utils/display_utils.py b/verifiers/utils/display_utils.py index 7f9b22769c..3314d8b488 100644 --- a/verifiers/utils/display_utils.py +++ b/verifiers/utils/display_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """ Shared utilities for Rich-based terminal displays. diff --git a/verifiers/utils/env_config_utils.py b/verifiers/utils/env_config_utils.py index 51e1114fd0..77cd0e6f1d 100644 --- a/verifiers/utils/env_config_utils.py +++ b/verifiers/utils/env_config_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + from collections.abc import Mapping from typing import cast diff --git a/verifiers/utils/env_utils.py b/verifiers/utils/env_utils.py index 06318cd9ab..fa81bf0c46 100644 --- a/verifiers/utils/env_utils.py +++ b/verifiers/utils/env_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import importlib import inspect import logging diff --git a/verifiers/utils/error_utils.py b/verifiers/utils/error_utils.py index 37ac1a4058..fda0ab89a6 100644 --- a/verifiers/utils/error_utils.py +++ b/verifiers/utils/error_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + from typing import Callable, TypeGuard, cast import verifiers as vf diff --git a/verifiers/utils/eval_display.py b/verifiers/utils/eval_display.py index 0f2c34c186..8df5101de3 100644 --- a/verifiers/utils/eval_display.py +++ b/verifiers/utils/eval_display.py @@ -1,3 +1,5 @@ +# ruff: noqa + """ Rich-based display for live multi-environment evaluation. diff --git a/verifiers/utils/eval_utils.py b/verifiers/utils/eval_utils.py index f33006dea4..e1cdabf72c 100644 --- a/verifiers/utils/eval_utils.py +++ b/verifiers/utils/eval_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import itertools import json diff --git a/verifiers/utils/heartbeat.py b/verifiers/utils/heartbeat.py index de581f0f50..1cdde2df7a 100644 --- a/verifiers/utils/heartbeat.py +++ b/verifiers/utils/heartbeat.py @@ -1,3 +1,5 @@ +# ruff: noqa + import logging import aiohttp diff --git a/verifiers/utils/install_utils.py b/verifiers/utils/install_utils.py index 018e8a3bf1..011e05ba16 100644 --- a/verifiers/utils/install_utils.py +++ b/verifiers/utils/install_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import importlib import logging import subprocess diff --git a/verifiers/utils/interception_utils.py b/verifiers/utils/interception_utils.py index a220e0f734..a50d435f02 100644 --- a/verifiers/utils/interception_utils.py +++ b/verifiers/utils/interception_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Utilities for intercepting API calls from agents running in sandboxes.""" import asyncio diff --git a/verifiers/utils/logging_utils.py b/verifiers/utils/logging_utils.py index 0f88b8d73c..edc1ceb488 100644 --- a/verifiers/utils/logging_utils.py +++ b/verifiers/utils/logging_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import sys diff --git a/verifiers/utils/message_utils.py b/verifiers/utils/message_utils.py index f6fc717c45..7a7cbb86d4 100644 --- a/verifiers/utils/message_utils.py +++ b/verifiers/utils/message_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import re diff --git a/verifiers/utils/metric_utils.py b/verifiers/utils/metric_utils.py index 41adea17e6..ec08385554 100644 --- a/verifiers/utils/metric_utils.py +++ b/verifiers/utils/metric_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import math from abc import ABC, abstractmethod from collections import defaultdict diff --git a/verifiers/utils/process_utils.py b/verifiers/utils/process_utils.py index 3fdc8dbd74..f241ddbf99 100644 --- a/verifiers/utils/process_utils.py +++ b/verifiers/utils/process_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Process lifecycle utilities.""" import logging diff --git a/verifiers/utils/save_utils.py b/verifiers/utils/save_utils.py index 8f59e8e01d..8f2acd4088 100644 --- a/verifiers/utils/save_utils.py +++ b/verifiers/utils/save_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import json import logging import os diff --git a/verifiers/utils/thread_utils.py b/verifiers/utils/thread_utils.py index e84de38196..4d516134f5 100644 --- a/verifiers/utils/thread_utils.py +++ b/verifiers/utils/thread_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import logging import threading diff --git a/verifiers/utils/threaded_sandbox_client.py b/verifiers/utils/threaded_sandbox_client.py index 25df03fc04..652c4cc97c 100644 --- a/verifiers/utils/threaded_sandbox_client.py +++ b/verifiers/utils/threaded_sandbox_client.py @@ -1,3 +1,5 @@ +# ruff: noqa + import asyncio import functools import logging diff --git a/verifiers/utils/usage_utils.py b/verifiers/utils/usage_utils.py index c8e3921b04..a4dbbed351 100644 --- a/verifiers/utils/usage_utils.py +++ b/verifiers/utils/usage_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + from collections.abc import Mapping, Sequence from types import MappingProxyType diff --git a/verifiers/utils/version_utils.py b/verifiers/utils/version_utils.py index 76e29f2c22..0ec6e56fb5 100644 --- a/verifiers/utils/version_utils.py +++ b/verifiers/utils/version_utils.py @@ -1,3 +1,5 @@ +# ruff: noqa + """Utilities for detecting verifiers and environment version/commit info.""" import importlib.metadata diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 55ab5998b7..2f72631e35 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -3,6 +3,7 @@ from pydantic_config import BaseConfig from verifiers.v1.acp import ACP +from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent from verifiers.v1.clients import ( BaseClientConfig, Client, @@ -12,9 +13,7 @@ TrainClientConfig, resolve_client, ) -from verifiers.v1.decorators import metric, reward, stop, tool from verifiers.v1.configs.agent import AgentConfig -from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent from verifiers.v1.configs.cli.env import ( ElasticPoolConfig, EnvServerConfig, @@ -22,8 +21,15 @@ pool_serve_kwargs, ) from verifiers.v1.configs.env import EnvConfig, default_agent_harness +from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.judge import JudgeConfig, Judges, JudgeSamplingConfig +from verifiers.v1.configs.retries import RetryConfig +from verifiers.v1.configs.task import TaskConfig +from verifiers.v1.configs.taskset import TasksetConfig +from verifiers.v1.decorators import metric, reward, stop, tool from verifiers.v1.env import Env from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig +from verifiers.v1.episode import Episode, WireEpisode from verifiers.v1.errors import ( EnvError, HarnessError, @@ -35,21 +41,19 @@ ToolsetError, TunnelError, ) -from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.graph import MessageNode from verifiers.v1.harness import Harness -from verifiers.v1.configs.judge import JudgeConfig, JudgeSamplingConfig, Judges from verifiers.v1.judge import Judge, JudgeResponse, JudgeView from verifiers.v1.judges import ( + Criterion, ReferenceJudge, ReferenceJudgeConfig, - Criterion, RubricJudge, RubricJudgeConfig, ) from verifiers.v1.loaders import ( default_harness_id, env_config_type, - resolve_env_config, environment_class, harness_config_type, import_environment, @@ -61,22 +65,14 @@ load_harness, load_judge, load_taskset, + resolve_env_config, task_type, taskset_config_type, ) -from verifiers.v1.scoring import ( - compare_stdout_results as compare_stdout_results, - extract_boxed_answer as extract_boxed_answer, - parse_judge_choice as parse_judge_choice, - parse_pytest_outcomes as parse_pytest_outcomes, - read_answer_file_or_last_reply as read_answer_file_or_last_reply, - verify_boxed_math_answer as verify_boxed_math_answer, -) -from verifiers.v1.configs.retries import RetryConfig -from verifiers.v1.utils.git import ( - PATCH_CAP_BYTES as PATCH_CAP_BYTES, - capture_patch as capture_patch, - resolve_head as resolve_head, +from verifiers.v1.mcp import ( + SharedToolsetConfig, + Toolset, + ToolsetConfig, ) from verifiers.v1.runtimes import ( DockerConfig, @@ -87,18 +83,27 @@ RuntimeInfo, SubprocessConfig, ) +from verifiers.v1.scoring import ( + compare_stdout_results as compare_stdout_results, +) +from verifiers.v1.scoring import ( + extract_boxed_answer as extract_boxed_answer, +) +from verifiers.v1.scoring import ( + parse_judge_choice as parse_judge_choice, +) +from verifiers.v1.scoring import ( + parse_pytest_outcomes as parse_pytest_outcomes, +) +from verifiers.v1.scoring import ( + read_answer_file_or_last_reply as read_answer_file_or_last_reply, +) +from verifiers.v1.scoring import ( + verify_boxed_math_answer as verify_boxed_math_answer, +) from verifiers.v1.state import State, StateT -from verifiers.v1.configs.task import TaskConfig from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout, WireTaskData -from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.taskset import Taskset -from verifiers.v1.mcp import ( - Toolset, - SharedToolsetConfig, - ToolsetConfig, -) -from verifiers.v1.graph import MessageNode -from verifiers.v1.episode import Episode, WireEpisode from verifiers.v1.trace import ( TRACE_VERSION, AgentInfo, @@ -119,9 +124,9 @@ WireTrace, ) from verifiers.v1.types import ( + ID, AssistantMessage, ContentPart, - ID, ImageUrlContentPart, ImageUrlSource, KeptTokens, @@ -136,13 +141,22 @@ TextContentPart, Tool, ToolCall, - TurnTokens, ToolMessage, + TurnTokens, Usage, UserMessage, ) +from verifiers.v1.utils.git import ( + PATCH_CAP_BYTES as PATCH_CAP_BYTES, +) +from verifiers.v1.utils.git import ( + capture_patch as capture_patch, +) +from verifiers.v1.utils.git import ( + resolve_head as resolve_head, +) -__all__ = [ +__all__ = [ # noqa: RUF022 - grouped by public API area # types "ID", "AssistantMessage", diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index d7ffda028f..f99158f3af 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -9,19 +9,18 @@ import asyncio import logging -from collections.abc import Callable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Iterator, Mapping from contextlib import asynccontextmanager, nullcontext from dataclasses import dataclass -from typing import AsyncIterator +from typing import Self - -from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig from verifiers.v1.clients import ( Client, EvalClientConfig, ModelContext, resolve_client, ) +from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig from verifiers.v1.harness import Harness from verifiers.v1.interception import Interception, InterceptionServer from verifiers.v1.mcp import SharedToolServer @@ -96,7 +95,7 @@ def _check_borrowed_placement( if task.data.image is None: return if isinstance(runtime.config, SubprocessConfig): - raise ValueError( + raise TypeError( f"task {task.data.idx!r} requires image {task.data.image!r}, but the " "borrowed runtime is subprocess-backed (no container); borrow a container " "box (e.g. agent.provision(task)) or drop the task's image" @@ -170,9 +169,8 @@ async def turn(self, message: str | Messages | None = None) -> Segment: prompted task speaks FIRST: take its opening reply with a bare `turn()` before answering. A `terminated` segment means the run ended instead of answering (the message went unconsumed).""" - async with self._lock: - async with self._gate or nullcontext(): - return await self._turn(message) + async with self._lock, self._gate or nullcontext(): + return await self._turn(message) async def _turn(self, message: str | Messages | None) -> Segment: if self._run.closed: @@ -220,11 +218,10 @@ async def _turn(self, message: str | Messages | None) -> Segment: async def close(self) -> Trace: """End the exchange and finish the rollout (idempotent): scoring and hooks run, then the finished trace returns (also on `interaction.trace`).""" - async with self._lock: - async with self._gate or nullcontext(): - if not self._run.closed and self._run.ok: - self.trace.stop("user_closed") - return await self._run.close() + async with self._lock, self._gate or nullcontext(): + if not self._run.closed and self._run.ok: + self.trace.stop("user_closed") + return await self._run.close() class Agent: @@ -290,7 +287,7 @@ def __init__( self._server: InterceptionServer | None = None self._warned_resources: set[tuple[str, str]] = set() - async def __aenter__(self) -> "Agent": + async def __aenter__(self) -> Self: if self._entered: raise RuntimeError("Agent is already entered; enter it once and share it") if self._closed: @@ -528,34 +525,34 @@ def _rollout_params( if self.timeout.rollout is not None else task.data.timeout.harness ) - return dict( - agent_config=self.config, - harness=self.harness, - ctx=self.ctx, - runtime_config=runtime_config, - setup_timeout=( + return { + "agent_config": self.config, + "harness": self.harness, + "ctx": self.ctx, + "runtime_config": runtime_config, + "setup_timeout": ( self.timeout.setup if self.timeout.setup is not None else task.data.timeout.setup ), - harness_timeout=cap_remote_harness_timeout( + "harness_timeout": cap_remote_harness_timeout( harness_timeout, runtime_config, task ), - finalize_timeout=( + "finalize_timeout": ( self.timeout.finalize if self.timeout.finalize is not None else task.data.timeout.finalize ), - scoring_timeout=( + "scoring_timeout": ( self.timeout.scoring if self.timeout.scoring is not None else task.data.timeout.scoring ), - limits=self.limits, - shared_tools=shared_tools, - interception=self._interception_for(run_is_local, task, shared_tools), - runtime=runtime, - ) + "limits": self.limits, + "shared_tools": shared_tools, + "interception": self._interception_for(run_is_local, task, shared_tools), + "runtime": runtime, + } @asynccontextmanager async def provision(self, task: Task | None = None) -> AsyncIterator[Runtime]: @@ -719,10 +716,8 @@ class Agents: field becomes an `Agent` under the field's name (`agents.solver`).""" def __init__(self, config, make: MakeAgent | None = None) -> None: - if make is None: - make = lambda _, spec: make_agent(spec) # noqa: E731 self._agents: dict[str, Agent] = { - name: make(name, value) + name: make_agent(value) if make is None else make(name, value) for name, value in agent_config_fields(config).items() } diff --git a/verifiers/v1/cli/dashboard/eval.py b/verifiers/v1/cli/dashboard/eval.py index 67f2ab7f02..f84392d40a 100644 --- a/verifiers/v1/cli/dashboard/eval.py +++ b/verifiers/v1/cli/dashboard/eval.py @@ -12,10 +12,9 @@ from rich.table import Table from rich.text import Text +from verifiers.utils.pricing_utils import format_cost_usd from verifiers.v1.cli.dashboard.base import live_view from verifiers.v1.cli.output import output_path -from verifiers.v1.utils.install import env_name -from verifiers.v1.utils.interrupt import cleaning_up from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.env import RunSlot from verifiers.v1.trace import Trace @@ -26,7 +25,8 @@ format_override, format_time, ) -from verifiers.utils.pricing_utils import format_cost_usd +from verifiers.v1.utils.install import env_name +from verifiers.v1.utils.interrupt import cleaning_up if TYPE_CHECKING: from verifiers.v1.push import PushState diff --git a/verifiers/v1/cli/debug.py b/verifiers/v1/cli/debug.py index eed30f8067..a4893cbabe 100644 --- a/verifiers/v1/cli/debug.py +++ b/verifiers/v1/cli/debug.py @@ -24,11 +24,11 @@ ) from verifiers.v1.configs.cli.debug import DebugConfig from verifiers.v1.decorators import invoke -from verifiers.v1.utils.compile import resolve_runtime_config from verifiers.v1.runtimes import ProgramResult, Runtime, make_runtime from verifiers.v1.state import state_cls from verifiers.v1.task import Task from verifiers.v1.trace import AgentInfo, Error, Trace, TraceTask +from verifiers.v1.utils.compile import resolve_runtime_config from verifiers.v1.utils.interrupt import install_interrupt from verifiers.v1.utils.logging import setup_logging @@ -165,7 +165,7 @@ async def run_timed( start = time.time() try: result = await asyncio.wait_for(action, config.timeout.total) - except Exception as e: + except Exception as e: # noqa: BLE001 - report any debug action failure return error_info(e, start, config.timeout.total, "debug action") return result_info(result, start) @@ -250,7 +250,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]: except asyncio.CancelledError as e: cancelled = True record_debug_error(trace, debug, e, setup_timeout, config.timeout.total) - except Exception as e: + except Exception as e: # noqa: BLE001 - persist any framework failure on the trace record_debug_error(trace, debug, e, setup_timeout, config.timeout.total) finally: trace.split_generation() diff --git a/verifiers/v1/cli/eval/main.py b/verifiers/v1/cli/eval/main.py index 682f76163a..0b3f66ae0f 100644 --- a/verifiers/v1/cli/eval/main.py +++ b/verifiers/v1/cli/eval/main.py @@ -7,8 +7,8 @@ from pydantic_config import cli import verifiers.v1 as vf -from verifiers.v1.utils.interrupt import install_interrupt -from verifiers.v1.utils.logging import setup_logging +from verifiers.v1.cli.eval.resume import load_resume_config, split_resume +from verifiers.v1.cli.eval.runner import run_eval from verifiers.v1.cli.output import output_path, write_config from verifiers.v1.cli.resolve import ( extract_id, @@ -17,9 +17,9 @@ references_config_file, with_positional_taskset, ) -from verifiers.v1.cli.eval.resume import load_resume_config, split_resume -from verifiers.v1.cli.eval.runner import run_eval from verifiers.v1.configs.cli.eval import EvalConfig +from verifiers.v1.utils.interrupt import install_interrupt +from verifiers.v1.utils.logging import setup_logging logger = logging.getLogger(__name__) diff --git a/verifiers/v1/cli/eval/resume.py b/verifiers/v1/cli/eval/resume.py index f12af98cf7..18bda68979 100644 --- a/verifiers/v1/cli/eval/resume.py +++ b/verifiers/v1/cli/eval/resume.py @@ -137,7 +137,8 @@ def parse(row: dict) -> Episode: episode = parse(row) if not verdict(episode): continue - except Exception: # malformed row: redo it + # A malformed row from any task/episode plugin is owed again. + except Exception: # noqa: BLE001, S112 continue good[key].append( (line if line.endswith(b"\n") else line + b"\n", episode) diff --git a/verifiers/v1/cli/eval/runner.py b/verifiers/v1/cli/eval/runner.py index 8eb1fb4e91..ca45870f8c 100644 --- a/verifiers/v1/cli/eval/runner.py +++ b/verifiers/v1/cli/eval/runner.py @@ -5,16 +5,16 @@ import logging import time -from verifiers.v1.clients import ModelContext, resolve_client -from verifiers.v1.configs.cli.eval import EvalConfig -from verifiers.v1.cli.eval import resume from verifiers.v1.cli.dashboard import dashboard +from verifiers.v1.cli.eval import resume from verifiers.v1.cli.output import ( append_episode, append_trace, output_path, save_config, ) +from verifiers.v1.clients import ModelContext, resolve_client +from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.env import Env, RunSlot from verifiers.v1.episode import Episode from verifiers.v1.trace import EvalRunInfo @@ -107,9 +107,9 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]: import multiprocessing as mp from functools import partial - from verifiers.v1.utils.logging import setup_logging from verifiers.v1.configs.cli.env import pool_serve_kwargs from verifiers.v1.serve import EnvClient, env_config_data, serve_env + from verifiers.v1.utils.logging import setup_logging legacy = config.is_legacy server_kwargs = ( diff --git a/verifiers/v1/cli/init.py b/verifiers/v1/cli/init.py index 3094573de3..b9c6e4b50c 100644 --- a/verifiers/v1/cli/init.py +++ b/verifiers/v1/cli/init.py @@ -17,7 +17,7 @@ def _names(name: str) -> tuple[str, str, str, str]: dash = name.strip().strip("/").replace("_", "-").lower() pkg = dash.replace("-", "_") - stem = pkg[:-3] if pkg.endswith("_v1") else pkg + stem = pkg.removesuffix("_v1") prefix = "".join(part[:1].upper() + part[1:] for part in stem.split("_") if part) if not prefix or not prefix[0].isalpha(): prefix = f"Env{prefix}" @@ -171,8 +171,10 @@ async def launch( def _readme(dash: str, pkg: str, *, add_tool: bool, add_harness: bool) -> str: layout = [ - f"- `{pkg}/taskset.py` — the task (`@reward` scoring + behavior) and the taskset: " - "`load` (data + prompts)." + ( + f"- `{pkg}/taskset.py` — the task (`@reward` scoring + behavior) and the taskset: " + "`load` (data + prompts)." + ) ] if add_tool: layout.append( diff --git a/verifiers/v1/cli/replay.py b/verifiers/v1/cli/replay.py index a535430221..4f8d16f725 100644 --- a/verifiers/v1/cli/replay.py +++ b/verifiers/v1/cli/replay.py @@ -27,10 +27,10 @@ save_config, write_config, ) +from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.configs.cli.replay import ReplayConfig from verifiers.v1.state import state_cls from verifiers.v1.task import Task, WireTaskData, task_data_cls -from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.trace import Trace from verifiers.v1.utils.interrupt import install_interrupt from verifiers.v1.utils.logging import setup_logging diff --git a/verifiers/v1/cli/serve.py b/verifiers/v1/cli/serve.py index 8269c89a64..6f5c5cdd5b 100644 --- a/verifiers/v1/cli/serve.py +++ b/verifiers/v1/cli/serve.py @@ -5,7 +5,6 @@ from pydantic_config import cli -from verifiers.v1.utils.logging import setup_logging from verifiers.v1.cli.resolve import ( extract_id, narrow_config, @@ -13,9 +12,10 @@ references_config_file, with_positional_taskset, ) -from verifiers.v1.configs.cli.serve import ServeConfig from verifiers.v1.configs.cli.env import pool_serve_kwargs +from verifiers.v1.configs.cli.serve import ServeConfig from verifiers.v1.serve import serve_env +from verifiers.v1.utils.logging import setup_logging USAGE = "usage: uv run serve [] [--env.id ] [--id (legacy)] [options] [@ file.toml]" diff --git a/verifiers/v1/cli/validate.py b/verifiers/v1/cli/validate.py index 7e446333d8..a889111736 100644 --- a/verifiers/v1/cli/validate.py +++ b/verifiers/v1/cli/validate.py @@ -20,11 +20,11 @@ ) from verifiers.v1.configs.cli.validate import ValidateConfig from verifiers.v1.decorators import invoke -from verifiers.v1.utils.compile import resolve_runtime_config from verifiers.v1.runtimes import make_runtime from verifiers.v1.state import state_cls from verifiers.v1.task import Task from verifiers.v1.trace import Trace, TraceTask +from verifiers.v1.utils.compile import resolve_runtime_config from verifiers.v1.utils.interrupt import install_interrupt from verifiers.v1.utils.logging import setup_logging @@ -103,7 +103,7 @@ async def _run_gold(task: Task, config: ValidateConfig) -> ResultRow: setup_timeout, ) valid = await asyncio.wait_for(task.validate(runtime), config.timeout.total) - except Exception as e: + except Exception as e: # noqa: BLE001 - validation reports plugin failures per task exc = e finally: try: @@ -138,7 +138,7 @@ async def _run_setup(task: Task, config: ValidateConfig) -> ResultRow: setup_timeout, ) valid = True - except Exception as e: + except Exception as e: # noqa: BLE001 - validation reports plugin failures per task exc = e finally: try: diff --git a/verifiers/v1/clients/__init__.py b/verifiers/v1/clients/__init__.py index 35728a3910..ab38dc7249 100644 --- a/verifiers/v1/clients/__init__.py +++ b/verifiers/v1/clients/__init__.py @@ -10,13 +10,13 @@ from verifiers.v1.clients.train import TrainClient __all__ = [ - "Client", - "ModelContext", "BaseClientConfig", + "Client", "ClientConfig", + "EvalClient", "EvalClientConfig", + "ModelContext", + "TrainClient", "TrainClientConfig", "resolve_client", - "EvalClient", - "TrainClient", ] diff --git a/verifiers/v1/clients/eval.py b/verifiers/v1/clients/eval.py index 6a7df3369c..2951bb23a8 100644 --- a/verifiers/v1/clients/eval.py +++ b/verifiers/v1/clients/eval.py @@ -12,8 +12,8 @@ change. Endpoint config (base url, api key, billing headers) comes from the client config. """ -from collections.abc import Mapping import re +from collections.abc import Mapping import httpx from pydantic import ValidationError diff --git a/verifiers/v1/clients/train.py b/verifiers/v1/clients/train.py index 6b7e7fad8e..785f05a923 100644 --- a/verifiers/v1/clients/train.py +++ b/verifiers/v1/clients/train.py @@ -13,9 +13,8 @@ from typing import Any from openai import AsyncOpenAI, OpenAIError -from renderers import RenderedTokens from renderers import OverlongPromptError as RendererOverlongPromptError -from renderers import RendererConfig +from renderers import RenderedTokens, RendererConfig from verifiers.v1.clients.client import SESSION_ID_HEADER, Client from verifiers.v1.dialects import FINISH_REASONS, ChatDialect, Dialect, parse_tools diff --git a/verifiers/v1/configs/cli/__init__.py b/verifiers/v1/configs/cli/__init__.py index 5e1de4beed..a5ee914a41 100644 --- a/verifiers/v1/configs/cli/__init__.py +++ b/verifiers/v1/configs/cli/__init__.py @@ -1,7 +1,7 @@ """The CLI entrypoints' run configs — each `uv run `'s parsed tree.""" -from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.configs.cli.debug import DebugConfig +from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.configs.cli.init import InitConfig from verifiers.v1.configs.cli.serve import ServeConfig from verifiers.v1.configs.cli.validate import ValidateConfig diff --git a/verifiers/v1/configs/cli/debug.py b/verifiers/v1/configs/cli/debug.py index 97df0bacde..3be8f1a031 100644 --- a/verifiers/v1/configs/cli/debug.py +++ b/verifiers/v1/configs/cli/debug.py @@ -7,8 +7,8 @@ from pydantic_config import BaseConfig from verifiers.v1.configs.cli.validate import CheckTimeoutConfig -from verifiers.v1.runtimes import DockerConfig, RuntimeConfig from verifiers.v1.configs.taskset import TasksetConfig +from verifiers.v1.runtimes import DockerConfig, RuntimeConfig class DebugConfig(BaseConfig): diff --git a/verifiers/v1/configs/cli/env.py b/verifiers/v1/configs/cli/env.py index ddba92462c..03f55e8efc 100644 --- a/verifiers/v1/configs/cli/env.py +++ b/verifiers/v1/configs/cli/env.py @@ -120,7 +120,7 @@ class EnvServerConfig(BaseConfig): """The environment — the run's `[env]` block: which env, its seed taskset, each agent, its knobs, and the run limits. Narrowed to the selected env's config class by the env id, else the taskset id.""" - pool: PoolConfig = ElasticPoolConfig() + pool: PoolConfig = Field(default_factory=ElasticPoolConfig) """Worker-pool sizing for the env server. `elastic` (default) starts at one worker and scales up on demand; `static` pre-spawns a fixed `num_workers`.""" # --- legacy (v0) backwards-compat ----------------------------------------- @@ -128,9 +128,9 @@ class by the env id, else the taskset id.""" """Classic (v0) env id (`name`, `org/name`, or `org/name@version` — installed from the hub on demand), loaded via `verifiers.load_environment` and run through the legacy bridge. Set this *instead of* `env.taskset` to run a v0 environment.""" - args: dict = {} + args: dict = Field(default_factory=dict) """Construction kwargs forwarded to `load_environment(id, **args)`.""" - extra_env_kwargs: dict = {} + extra_env_kwargs: dict = Field(default_factory=dict) """Post-load kwargs applied to the v0 env via `env.set_kwargs(**extra_env_kwargs)` (e.g. `max_total_completion_tokens`, `max_seq_len`, `timeout_seconds`) — typically auto-populated by the orchestrator, distinct from the `args` passed at construction.""" diff --git a/verifiers/v1/configs/cli/validate.py b/verifiers/v1/configs/cli/validate.py index 4197703028..7fad910b17 100644 --- a/verifiers/v1/configs/cli/validate.py +++ b/verifiers/v1/configs/cli/validate.py @@ -3,8 +3,8 @@ from pydantic import AliasChoices, Field, SerializeAsAny, model_validator from pydantic_config import BaseConfig -from verifiers.v1.runtimes import DockerConfig, RuntimeConfig from verifiers.v1.configs.taskset import TasksetConfig +from verifiers.v1.runtimes import DockerConfig, RuntimeConfig class CheckTimeoutConfig(BaseConfig): diff --git a/verifiers/v1/configs/retries.py b/verifiers/v1/configs/retries.py index 85f6e8bc14..488a4e4860 100644 --- a/verifiers/v1/configs/retries.py +++ b/verifiers/v1/configs/retries.py @@ -11,7 +11,7 @@ class RetryConfig(BaseConfig): max_retries: int = Field(0, ge=0) """Whole-rollout retries beyond the first attempt. Off by default — the SDKs already retry transient per-call faults; rerunning a whole trajectory is opt-in.""" - include: list[str] = [] + include: list[str] = Field(default_factory=list) """Only retry errors whose type is listed. Empty = retry anything not excluded.""" - exclude: list[str] = [] + exclude: list[str] = Field(default_factory=list) """Never retry errors whose type is listed (wins over `include`).""" diff --git a/verifiers/v1/configs/task.py b/verifiers/v1/configs/task.py index 889cce1067..13dcb62791 100644 --- a/verifiers/v1/configs/task.py +++ b/verifiers/v1/configs/task.py @@ -2,7 +2,7 @@ from __future__ import annotations -from pydantic import model_validator +from pydantic import Field, model_validator from pydantic_config import BaseConfig from verifiers.v1.configs.judge import Judges, check_judges, resolve_judges @@ -16,7 +16,7 @@ class TaskConfig(BaseConfig): Load-time dataset settings belong on `TasksetConfig` instead. """ - judges: Judges = [] + judges: Judges = Field(default_factory=list) """Judge plugins run after task rewards, set through `--env.taskset.task.judges`.""" @model_validator(mode="before") diff --git a/verifiers/v1/decorators.py b/verifiers/v1/decorators.py index aecc218644..15bce8b35d 100644 --- a/verifiers/v1/decorators.py +++ b/verifiers/v1/decorators.py @@ -2,7 +2,8 @@ import asyncio import inspect -from typing import Any, Callable, TypeVar, overload +from collections.abc import Callable +from typing import Any, TypeVar, overload F = TypeVar("F", bound=Callable[..., Any]) diff --git a/verifiers/v1/dialects/__init__.py b/verifiers/v1/dialects/__init__.py index 4e828061bf..c195b31c72 100644 --- a/verifiers/v1/dialects/__init__.py +++ b/verifiers/v1/dialects/__init__.py @@ -14,11 +14,11 @@ the wire format from the route a request arrived on.""" __all__ = [ - "Dialect", "DIALECTS", "FINISH_REASONS", "AnthropicDialect", "ChatDialect", + "Dialect", "ResponsesDialect", "StreamParser", "parse_message", diff --git a/verifiers/v1/dialects/chat.py b/verifiers/v1/dialects/chat.py index ed9a1b8514..23277f4c4f 100644 --- a/verifiers/v1/dialects/chat.py +++ b/verifiers/v1/dialects/chat.py @@ -8,7 +8,8 @@ import time from collections.abc import Mapping -from dataclasses import dataclass, field as dataclass_field +from dataclasses import dataclass +from dataclasses import field as dataclass_field from typing import Any from openai.types.chat import ChatCompletion diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 0564533e96..0fcee6d171 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -12,31 +12,29 @@ TypeVar, ) - from verifiers.v1.agent import Agent, Agents, _EpisodeAgent +from verifiers.v1.clients import Client, ClientConfig, ModelContext, resolve_client from verifiers.v1.configs.agent import AgentConfig from verifiers.v1.configs.env import ( EnvConfig, _declared_agent_configs, default_agent_harness, ) +from verifiers.v1.episode import Episode +from verifiers.v1.errors import EnvError, boundary from verifiers.v1.harness import Harness, HarnessConfig -from verifiers.v1.clients import Client, ClientConfig, ModelContext, resolve_client from verifiers.v1.interception import ( Interception, make_interception, requires_tunnel, ) +from verifiers.v1.mcp import SharedToolServer, serve_shared from verifiers.v1.retries import run_episode_with_retry from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local -from verifiers.v1.errors import EnvError, boundary from verifiers.v1.task import Task, resolve_server_config -from verifiers.v1.episode import Episode from verifiers.v1.trace import Error, Trace from verifiers.v1.utils.generic import generic_type from verifiers.v1.utils.memory import trim_memory_periodically -from verifiers.v1.mcp import SharedToolServer, serve_shared - logger = logging.getLogger(__name__) @@ -266,7 +264,7 @@ async def run_episode( f"{type(self).__name__}.run() ran no agent — every " "episode must carry at least one run" ) - except Exception as e: + except Exception as e: # noqa: BLE001 - episode boundary records every hook failure # Only the deadline's expiry: inner TimeoutErrors became EnvError already. if isinstance(e, TimeoutError): e = TimeoutError( @@ -280,7 +278,7 @@ async def run_episode( async with asyncio.timeout(self.config.timeout.finalize): async with boundary(EnvError, f"{type(self).__name__}.finalize()"): await self.finalize(task, episode) - except Exception as e: + except Exception as e: # noqa: BLE001 - episode boundary records every hook failure # As above: a TimeoutError here is the deadline's own expiry. if isinstance(e, TimeoutError): e = TimeoutError( diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index 3ad1a66e70..60a8dd6b9a 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -17,7 +17,7 @@ import tomllib from pathlib import Path -from pydantic import field_validator +from pydantic import Field, field_validator import verifiers.v1 as vf from verifiers.v1.types import StrictBaseModel @@ -45,7 +45,7 @@ class Criterion(StrictBaseModel): text: str weight: float = 1.0 """The criterion's share of the reward.""" - choices: list[str] = ["no", "yes"] + choices: list[str] = Field(default_factory=lambda: ["no", "yes"]) """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest evenly spaced by rank. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates.""" @@ -287,7 +287,7 @@ def _check_agents(self) -> None: "(--env.taskset.task.judges), not an agent." ) if isinstance(self.config.solver.runtime, vf.SubprocessConfig): - raise ValueError( + raise TypeError( "agentic-judge plays its judge in the solver's box, but the solver " "(which provisions it) resolves to the subprocess runtime; use " "--env.solver.runtime.type docker or prime" @@ -308,7 +308,7 @@ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: solution, verdict = by_agent["solver"], by_agent["judge"] data = verdict.info.get("verdict") if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list): - raise ValueError( + raise TypeError( f"no verdicts on the judge's trace (expected {VERDICT_FILE} with a " '"verdicts" list)' ) @@ -317,7 +317,7 @@ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: answers: dict[str, str] = {} for entry in data["verdicts"]: if not isinstance(entry, dict): - raise ValueError(f"verdict entry {entry!r} is not an object") + raise TypeError(f"verdict entry {entry!r} is not an object") name = str(entry.get("name")) if name in answers: # Contradictory duplicates must not collapse to whichever came last. diff --git a/verifiers/v1/envs/single_agent/env.py b/verifiers/v1/envs/single_agent/env.py index 401303750b..537e21ad4a 100644 --- a/verifiers/v1/envs/single_agent/env.py +++ b/verifiers/v1/envs/single_agent/env.py @@ -5,8 +5,8 @@ eval is exactly this env (one `agent` seat, one nameless trace per episode). """ -from verifiers.v1.configs.agent import AgentConfig from verifiers.v1.agent import Agents +from verifiers.v1.configs.agent import AgentConfig from verifiers.v1.configs.env import EnvConfig from verifiers.v1.env import Env from verifiers.v1.task import Task diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index 9099ace17f..ccce076402 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -5,9 +5,9 @@ from pydantic import Field +from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData -from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.trace import AgentConfigT, Error, Trace from verifiers.v1.types import StrictBaseModel diff --git a/verifiers/v1/gepa/adapter.py b/verifiers/v1/gepa/adapter.py index 991bd3715b..c9f6f42b5c 100644 --- a/verifiers/v1/gepa/adapter.py +++ b/verifiers/v1/gepa/adapter.py @@ -9,17 +9,17 @@ """ import asyncio -from collections.abc import Awaitable +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass, field -from typing import Any, Callable, Mapping, Sequence +from typing import Any from gepa.core.adapter import EvaluationBatch from pydantic_core import to_jsonable_python from verifiers.v1.clients import ModelContext from verifiers.v1.env import Env -from verifiers.v1.task import Task from verifiers.v1.episode import Episode +from verifiers.v1.task import Task Candidate = dict[str, str] @@ -87,7 +87,7 @@ async def _run_batch(self, batch: list[int], system_prompt: str) -> list[Episode def make_reflective_dataset( self, - candidate: Candidate, # noqa: ARG002 - required by GEPA's adapter protocol + candidate: Candidate, eval_batch: EvaluationBatch[Episode, Episode], components_to_update: list[str], ) -> Mapping[str, Sequence[Mapping[str, Any]]]: diff --git a/verifiers/v1/gepa/reflection.py b/verifiers/v1/gepa/reflection.py index e606fcaef9..90c6735127 100644 --- a/verifiers/v1/gepa/reflection.py +++ b/verifiers/v1/gepa/reflection.py @@ -5,10 +5,10 @@ """ import asyncio -from typing import Callable +from collections.abc import Callable -from verifiers.v1.gepa.config import GEPAConfig from verifiers.v1.configs.judge import JudgeConfig +from verifiers.v1.gepa.config import GEPAConfig from verifiers.v1.judge import Judge diff --git a/verifiers/v1/gepa/runner.py b/verifiers/v1/gepa/runner.py index 93b4e2e7b6..86a8b9ca90 100644 --- a/verifiers/v1/gepa/runner.py +++ b/verifiers/v1/gepa/runner.py @@ -14,6 +14,7 @@ from verifiers.v1.cli.output import append_episode, output_path, save_config from verifiers.v1.clients import ModelContext, resolve_client from verifiers.v1.env import Env +from verifiers.v1.episode import Episode from verifiers.v1.gepa.adapter import GEPAAdapter from verifiers.v1.gepa.config import GEPAConfig from verifiers.v1.gepa.dataset import ( @@ -21,7 +22,6 @@ split_tasks, ) from verifiers.v1.gepa.reflection import build_reflection_lm -from verifiers.v1.episode import Episode logger = logging.getLogger(__name__) @@ -91,20 +91,20 @@ async def on_complete(episode: Episode) -> None: on_complete=on_complete, reflection_columns=config.reflection_columns, ) - optimize_kwargs: dict = dict( - seed_candidate={"system_prompt": seed_prompt}, - trainset=[task.data.idx for task in train_tasks], - valset=[task.data.idx for task in val_tasks], - adapter=adapter, - reflection_lm=reflection_lm, - max_metric_calls=config.max_total_rollouts, - reflection_minibatch_size=config.reflection_minibatch_size, - run_dir=str(run_dir) if run_dir is not None else None, - seed=config.seed, - display_progress_bar=False, - skip_perfect_score=False, - logger=_GEPALog(), - ) + optimize_kwargs: dict = { + "seed_candidate": {"system_prompt": seed_prompt}, + "trainset": [task.data.idx for task in train_tasks], + "valset": [task.data.idx for task in val_tasks], + "adapter": adapter, + "reflection_lm": reflection_lm, + "max_metric_calls": config.max_total_rollouts, + "reflection_minibatch_size": config.reflection_minibatch_size, + "run_dir": str(run_dir) if run_dir is not None else None, + "seed": config.seed, + "display_progress_bar": False, + "skip_perfect_score": False, + "logger": _GEPALog(), + } return optimize(**optimize_kwargs) finally: loop.run_until_complete(serving.__aexit__(None, None, None)) diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index 8f8eefe691..78fdb3a0a3 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -6,11 +6,10 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar - from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.decorators import discover_decorated, invoke_all from verifiers.v1.errors import HarnessError, boundary -from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.runtimes import ProgramResult, Runtime from verifiers.v1.task import TaskData from verifiers.v1.types import Messages @@ -65,7 +64,7 @@ def resolve_prompt( if system is None or self.APPENDS_SYSTEM_PROMPT: return system if self.APPENDS_SYSTEM_PROMPT else None, prompt if not isinstance(prompt, str): - raise ValueError( + raise TypeError( f"Harness {self.config.id!r} cannot fold a system prompt into a " f"{'Messages' if prompt is not None else 'None'} prompt; set " "APPENDS_SYSTEM_PROMPT to emit it as a system message." diff --git a/verifiers/v1/harnesses/__init__.py b/verifiers/v1/harnesses/__init__.py index ce1b6117a9..fe5166209c 100644 --- a/verifiers/v1/harnesses/__init__.py +++ b/verifiers/v1/harnesses/__init__.py @@ -24,14 +24,14 @@ "CodexHarnessConfig", "KimiCodeHarness", "KimiCodeHarnessConfig", + "MiniSWEAgentHarness", + "MiniSWEAgentHarnessConfig", "NullHarness", "NullHarnessConfig", "PiHarness", "PiHarnessConfig", "PoolHarness", "PoolHarnessConfig", - "MiniSWEAgentHarness", - "MiniSWEAgentHarnessConfig", "RLMHarness", "RLMHarnessConfig", "Terminus2Harness", diff --git a/verifiers/v1/harnesses/bash/harness.py b/verifiers/v1/harnesses/bash/harness.py index 49b0c6efc9..b94989e535 100644 --- a/verifiers/v1/harnesses/bash/harness.py +++ b/verifiers/v1/harnesses/bash/harness.py @@ -2,13 +2,13 @@ import os from pathlib import Path -from verifiers.v1.configs.harness import HarnessConfig -from verifiers.v1.harness import Harness from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.dialects.chat import message_to_wire +from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime -from verifiers.v1.trace import Trace from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text() diff --git a/verifiers/v1/harnesses/bash/program.py b/verifiers/v1/harnesses/bash/program.py index ff17096ad8..aa9326c806 100644 --- a/verifiers/v1/harnesses/bash/program.py +++ b/verifiers/v1/harnesses/bash/program.py @@ -129,17 +129,21 @@ def run_search(query: str, api_key: str, num_results: int = 5) -> str: response.raise_for_status() organic = response.json().get("organic") or [] return format_results(organic[:num_results], query) - except Exception as e: + except Exception as e: # noqa: BLE001 - tool failures are returned to the model return f"search failed ({e}). Try again or rephrase the query." def run_bash(command: str) -> str: try: result = subprocess.run( - ["bash", "-c", command], capture_output=True, text=True, timeout=3600 + ["bash", "-c", command], + capture_output=True, + text=True, + timeout=3600, + check=False, ) return result.stdout + result.stderr - except Exception as e: + except Exception as e: # noqa: BLE001 - tool failures are returned to the model return f"error: {e}" @@ -161,14 +165,14 @@ def run_edit(path: str, old_str: str, new_str: str) -> str: # error as a tool result instead of letting it abort the chat loop. try: content = filepath.read_text() - except Exception as e: + except Exception as e: # noqa: BLE001 - tool failures are returned to the model return f"error: could not read {path}: {e}" count = content.count(old_str) if count != 1: return f"error: old_str must appear exactly once in {path} (found {count})" try: filepath.write_text(content.replace(old_str, new_str, 1)) - except Exception as e: + except Exception as e: # noqa: BLE001 - tool failures are returned to the model return f"error: could not write {path}: {e}" return f"Edited {path}" diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index eb8f9cc8ac..af55a3acc8 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -12,9 +12,9 @@ from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime +from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace from verifiers.v1.types import TextContentPart -from verifiers.v1.task import TaskData logger = logging.getLogger(__name__) @@ -176,7 +176,7 @@ async def resume( ) for part in parts: if not isinstance(part, TextContentPart): - raise ValueError( + raise TypeError( "codex resume supports text user turns only (images go in " "the opening prompt)" ) diff --git a/verifiers/v1/harnesses/mini_swe_agent/harness.py b/verifiers/v1/harnesses/mini_swe_agent/harness.py index 3925e3ae24..b52c96aa6a 100644 --- a/verifiers/v1/harnesses/mini_swe_agent/harness.py +++ b/verifiers/v1/harnesses/mini_swe_agent/harness.py @@ -4,8 +4,8 @@ from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime -from verifiers.v1.trace import Trace from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text() diff --git a/verifiers/v1/harnesses/null/harness.py b/verifiers/v1/harnesses/null/harness.py index 96f85d7d7e..2da3eb8378 100644 --- a/verifiers/v1/harnesses/null/harness.py +++ b/verifiers/v1/harnesses/null/harness.py @@ -1,13 +1,13 @@ import json from pathlib import Path -from verifiers.v1.configs.harness import HarnessConfig -from verifiers.v1.harness import Harness from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.dialects.chat import message_to_wire +from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime -from verifiers.v1.trace import Trace from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text() diff --git a/verifiers/v1/harnesses/pi/harness.py b/verifiers/v1/harnesses/pi/harness.py index 6d1313e6ef..a63ca85349 100644 --- a/verifiers/v1/harnesses/pi/harness.py +++ b/verifiers/v1/harnesses/pi/harness.py @@ -30,10 +30,12 @@ ACP_COMMAND = [ "sh", "-c", - f'export {HOME_VAR}="$HOME"; ' - 'PI_CODING_AGENT_DIR="$PWD/$PI_CODING_AGENT_DIR"; ' - 'export PI_CODING_AGENT_DIR HOME="$PI_CODING_AGENT_DIR"; ' - f'export PATH="{PI_DIR}/node/bin:$PATH"; exec {ACP_BIN}', + ( + f'export {HOME_VAR}="$HOME"; ' + 'PI_CODING_AGENT_DIR="$PWD/$PI_CODING_AGENT_DIR"; ' + 'export PI_CODING_AGENT_DIR HOME="$PI_CODING_AGENT_DIR"; ' + f'export PATH="{PI_DIR}/node/bin:$PATH"; exec {ACP_BIN}' + ), ] INSTALL = r""" diff --git a/verifiers/v1/harnesses/pool/harness.py b/verifiers/v1/harnesses/pool/harness.py index 9e62d723ab..0382bd38d4 100644 --- a/verifiers/v1/harnesses/pool/harness.py +++ b/verifiers/v1/harnesses/pool/harness.py @@ -86,11 +86,13 @@ async def launch( command = [ "sh", "-c", - f'export HOME="$PWD/{pool_home}/home" ' - f'XDG_CONFIG_HOME="$PWD/{pool_home}/config" ' - f'XDG_STATE_HOME="$PWD/{pool_home}/state"; ' - f"exec {POOL_DIR.format(version=self.config.version)}/pool acp " - f"--sandbox disabled --settings {settings}", + ( + f'export HOME="$PWD/{pool_home}/home" ' + f'XDG_CONFIG_HOME="$PWD/{pool_home}/config" ' + f'XDG_STATE_HOME="$PWD/{pool_home}/state"; ' + f"exec {POOL_DIR.format(version=self.config.version)}/pool acp " + f"--sandbox disabled --settings {settings}" + ), ] return await POOL_ACP.run( runtime, diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index 3884dd3562..9bc4194301 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -6,16 +6,16 @@ import shlex from typing import Literal -from pydantic import model_validator +from pydantic import Field, model_validator -from verifiers.v1.configs.harness import HarnessConfig -from verifiers.v1.harness import Harness from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.decorators import metric from verifiers.v1.dialects.chat import message_to_wire +from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime -from verifiers.v1.trace import Trace from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace logger = logging.getLogger(__name__) @@ -35,7 +35,7 @@ class RLMHarnessConfig(HarnessConfig): """Git ref (branch, tag, or commit) of rlm to install.""" max_depth: int = 0 """Recursion depth rlm may spawn sub-harnesses to (RLM_MAX_DEPTH).""" - builtin_skills: list[BuiltinSkill] = [] + builtin_skills: list[BuiltinSkill] = Field(default_factory=list) """Built-in rlm skills to enable (RLM_SKILLS), e.g. `["edit"]`; empty enables none. The tool set is fixed (ipython); the base `skills` field takes SKILL.md paths.""" summarize_at_tokens: int | tuple[int, int] | None = None diff --git a/verifiers/v1/harnesses/terminus_2/harness.py b/verifiers/v1/harnesses/terminus_2/harness.py index e92992e5f4..6e9feb7266 100644 --- a/verifiers/v1/harnesses/terminus_2/harness.py +++ b/verifiers/v1/harnesses/terminus_2/harness.py @@ -5,8 +5,8 @@ from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime -from verifiers.v1.trace import Trace from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text() logger = logging.getLogger(__name__) diff --git a/verifiers/v1/harnesses/terminus_2/program.py b/verifiers/v1/harnesses/terminus_2/program.py index 9eee3cf7b1..916c992b15 100644 --- a/verifiers/v1/harnesses/terminus_2/program.py +++ b/verifiers/v1/harnesses/terminus_2/program.py @@ -28,7 +28,8 @@ async def exec( user: str | int | None = None, ) -> ExecResult: _ = user - result = subprocess.run( + result = await asyncio.to_thread( + subprocess.run, command, shell=True, cwd=cwd, @@ -36,6 +37,7 @@ async def exec( capture_output=True, text=True, timeout=timeout_sec, + check=False, ) return ExecResult( stdout=result.stdout, diff --git a/verifiers/v1/interception/__init__.py b/verifiers/v1/interception/__init__.py index f1c4663825..9d5646f87e 100644 --- a/verifiers/v1/interception/__init__.py +++ b/verifiers/v1/interception/__init__.py @@ -67,16 +67,16 @@ def make_interception( __all__ = [ - "Interception", - "Slot", - "make_interception", - "requires_tunnel", - "InterceptionServer", - "StaticInterceptionPool", - "ElasticInterceptionPool", "BaseInterceptionConfig", + "ElasticInterceptionPool", + "ElasticInterceptionPoolConfig", + "Interception", "InterceptionConfig", + "InterceptionServer", "InterceptionServerConfig", + "Slot", + "StaticInterceptionPool", "StaticInterceptionPoolConfig", - "ElasticInterceptionPoolConfig", + "make_interception", + "requires_tunnel", ] diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 13a7e300f4..b9e5574d19 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -33,9 +33,9 @@ from pydantic import TypeAdapter, ValidationError from pydantic_core import PydanticSerializationError, from_json, to_json +from verifiers.v1 import graph from verifiers.v1.dialects import DIALECTS, Dialect from verifiers.v1.dialects.base import is_sse_done_event -from verifiers.v1 import graph from verifiers.v1.errors import ( OverlongPromptError, ProviderError, @@ -382,7 +382,7 @@ def serve(response: Response) -> web.Response: refused = await session.refused() except RolloutError as e: return self._fail(session, dialect, e) - except Exception as e: + except Exception as e: # noqa: BLE001 - surface any task hook failure return self._fail( session, dialect, @@ -456,7 +456,7 @@ def serve(response: Response) -> web.Response: dialect.error_body(str(e)), status=getattr(e, "status_code", 502), ) - except Exception as e: # surface to the program as an API error + except Exception as e: # noqa: BLE001 - surface as an API error error = e logger.warning( "model call failed: id=%s %s: %s", @@ -515,7 +515,7 @@ async def _stream( refused = await session.refused() except RolloutError as e: return self._fail(session, dialect, e) - except Exception as e: + except Exception as e: # noqa: BLE001 - surface any task hook failure return self._fail( session, dialect, TaskError(f"@stop failed: {type(e).__name__}: {e}") ) @@ -563,7 +563,7 @@ async def _stream( return web.json_response( dialect.error_body(str(e)), status=getattr(e, "status_code", 502) ) - except Exception as e: # surface to the program as an API error + except Exception as e: # noqa: BLE001 - surface as an API error error = e logger.warning("model call failed: id=%s %s", session.trace.id, e) return web.json_response(dialect.error_body(str(e)), status=502) @@ -616,7 +616,7 @@ async def _stream( if on_done is not None and is_sse_done_event(chunk): on_done() feed_event(chunk) - except Exception as e: + except Exception as e: # noqa: BLE001 - defer parser failure parser_error = e # forwarded after the turn is committed, below deferred.append(chunk) @@ -625,7 +625,7 @@ async def _stream( if parser_error is None: try: feed_event(chunk) - except Exception as e: + except Exception as e: # noqa: BLE001 - defer parser failure parser_error = e except ConnectionResetError as e: # The harness went away mid-stream; the provider exchange still happened. @@ -700,7 +700,7 @@ async def handle_aux( return web.json_response( dialect.error_body(str(e)), status=getattr(e, "status_code", 502) ) - except Exception as e: + except Exception as e: # noqa: BLE001 - surface auxiliary relay failures logger.warning("aux call failed: id=%s %s", session.trace.id, e) return web.json_response(dialect.error_body(str(e)), status=502) return web.json_response(result) diff --git a/verifiers/v1/interception/tunnel/__init__.py b/verifiers/v1/interception/tunnel/__init__.py index e60dbc8234..c2043bc39d 100644 --- a/verifiers/v1/interception/tunnel/__init__.py +++ b/verifiers/v1/interception/tunnel/__init__.py @@ -20,12 +20,12 @@ def make_tunnel(config: TunnelConfig) -> Tunnel: __all__ = [ - "Tunnel", "BaseTunnelConfig", - "TunnelConfig", - "PrimeTunnel", - "PrimeTunnelConfig", "CustomTunnel", "CustomTunnelConfig", + "PrimeTunnel", + "PrimeTunnelConfig", + "Tunnel", + "TunnelConfig", "make_tunnel", ] diff --git a/verifiers/v1/judge.py b/verifiers/v1/judge.py index 53d224e408..afc3136ba8 100644 --- a/verifiers/v1/judge.py +++ b/verifiers/v1/judge.py @@ -49,8 +49,8 @@ async def correct(self, trace) -> float: from __future__ import annotations import re -from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Callable, Generic, Literal, cast +from collections.abc import Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, Generic, Literal, cast from pydantic import BaseModel from typing_extensions import TypeVar @@ -62,8 +62,8 @@ async def correct(self, trace) -> float: ) from verifiers.v1.dialects.chat import message_to_wire from verifiers.v1.scoring import parse_judge_choice -from verifiers.v1.utils.generic import generic_type from verifiers.v1.types import Messages, StrictBaseModel, Usage +from verifiers.v1.utils.generic import generic_type if TYPE_CHECKING: from verifiers.v1.task import TaskData @@ -81,7 +81,7 @@ class JudgeResponse(StrictBaseModel, Generic[ParsedT]): JudgeView = Literal["last_reply", "full_trace"] -def judge_question(task: "TaskData", question_field: str) -> str: +def judge_question(task: TaskData, question_field: str) -> str: if not question_field: return task.prompt_text question = getattr(task, question_field, None) @@ -94,7 +94,7 @@ def judge_question(task: "TaskData", question_field: str) -> str: return str(question) -def judge_response(trace: "Trace", view: JudgeView) -> str: +def judge_response(trace: Trace, view: JudgeView) -> str: return trace.transcript if view == "full_trace" else trace.last_reply @@ -147,9 +147,7 @@ def build_messages(self, **fields: Any) -> str | Messages: pattern = re.compile(r"\{(" + "|".join(map(re.escape, fields)) + r")\}") return pattern.sub(lambda m: str(fields[m.group(1)]), template) - async def score( - self, task: "TaskData", trace: "Trace" - ) -> float | Mapping[str, float]: + async def score(self, task: TaskData, trace: Trace) -> float | Mapping[str, float]: raise NotImplementedError( f"{type(self).__name__} implements no `score`, so it can't be plugged via " "`taskset.task.judges`; implement `score` (see verifiers.v1.judges for examples) or " @@ -165,7 +163,7 @@ async def complete( self, messages: str | Messages, *, - trace: "Trace | None" = None, + trace: Trace | None = None, schema: type[BaseModel] | None = None, parse: Callable[[JudgeResponse[Any]], Any] | None = None, **sampling: Any, @@ -216,7 +214,7 @@ async def complete( trace.record_judge(response) async def evaluate( - self, *, trace: "Trace | None" = None, **fields: Any + self, *, trace: Trace | None = None, **fields: Any ) -> JudgeResponse[ParsedT]: messages = self.build_messages(**fields) return await self.complete( diff --git a/verifiers/v1/judges/__init__.py b/verifiers/v1/judges/__init__.py index 36135b2f3e..cd8f624c2f 100644 --- a/verifiers/v1/judges/__init__.py +++ b/verifiers/v1/judges/__init__.py @@ -2,9 +2,9 @@ from verifiers.v1.judges.rubric import Criterion, RubricJudge, RubricJudgeConfig __all__ = [ + "Criterion", "ReferenceJudge", "ReferenceJudgeConfig", - "Criterion", "RubricJudge", "RubricJudgeConfig", ] diff --git a/verifiers/v1/judges/rubric.py b/verifiers/v1/judges/rubric.py index c7a84856c0..0419454846 100644 --- a/verifiers/v1/judges/rubric.py +++ b/verifiers/v1/judges/rubric.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import cast -from pydantic import field_validator +from pydantic import Field, field_validator from verifiers.v1.configs.judge import JudgeConfig from verifiers.v1.judge import Judge, JudgeView, judge_question, judge_response @@ -62,7 +62,7 @@ class Criterion(StrictBaseModel): text: str weight: float = 1.0 """The criterion's share of the reward (overridable per name via `weights` in config).""" - choices: list[str] = ["no", "yes"] + choices: list[str] = Field(default_factory=lambda: ["no", "yes"]) """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest evenly spaced by rank. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates.""" @@ -82,7 +82,7 @@ class RubricJudgeConfig(JudgeConfig): path: Path """A `.toml` or `.json` file containing a `criteria` list. Relative paths resolve against the evaluation's working directory.""" - weights: dict[str, float] = {} + weights: dict[str, float] = Field(default_factory=dict) """Per-criterion weight overrides by criterion name (config wins over the file).""" question_field: str = "" """Task field to fill the prompt's `{question}`; empty = the task's prompt rendered as @@ -187,12 +187,12 @@ def render(c: Criterion) -> str: f"{fence}\n{answer}\n{fence}\n" ) - fields = dict( - question=judge_question(task, self.config.question_field), - response=judge_response(trace, self.config.view), - criteria="\n".join(render(c) for c in batch), - reference=reference, - ) + fields = { + "question": judge_question(task, self.config.question_field), + "response": judge_response(trace, self.config.view), + "criteria": "\n".join(render(c) for c in batch), + "reference": reference, + } if self.config.structured_output: result = await self.evaluate(trace=trace, **fields) verdicts = cast(RubricVerdicts, result.parsed).verdicts diff --git a/verifiers/v1/legacy.py b/verifiers/v1/legacy.py index 32f5866eaf..79228c63b3 100644 --- a/verifiers/v1/legacy.py +++ b/verifiers/v1/legacy.py @@ -22,7 +22,9 @@ import zmq.asyncio from pydantic import ValidationError +from verifiers.v1 import graph from verifiers.v1.clients.config import ClientConfig, TrainClientConfig +from verifiers.v1.episode import Episode from verifiers.v1.serve.server import EnvServer from verifiers.v1.serve.types import ( RunGroupRequest, @@ -31,8 +33,6 @@ RunResponse, ) from verifiers.v1.task import WireTaskData -from verifiers.v1 import graph -from verifiers.v1.episode import Episode from verifiers.v1.trace import ( Error, GenerationSpan, @@ -504,7 +504,6 @@ async def run_legacy_eval(config) -> list[Episode]: import asyncio from verifiers import load_environment - from verifiers.v1.cli.output import append_trace, save_config from verifiers.v1.utils.install import ensure_installed, env_name from verifiers.v1.utils.sampling import sample diff --git a/verifiers/v1/loaders.py b/verifiers/v1/loaders.py index 0049c0c4fb..edfaebf9a1 100644 --- a/verifiers/v1/loaders.py +++ b/verifiers/v1/loaders.py @@ -4,31 +4,30 @@ import importlib import importlib.util import pkgutil +from collections.abc import Callable from types import ModuleType -from typing import Callable from pydantic import ValidationError from pydantic_config import BaseConfig from verifiers.v1.configs.env import EnvConfig +from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.judge import JudgeConfig +from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.env import Env -from verifiers.v1.utils.generic import prefix_validation_error from verifiers.v1.envs.single_agent import SingleAgentEnv -from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.harness import Harness -from verifiers.v1.configs.judge import JudgeConfig from verifiers.v1.judge import Judge, judge_config_cls -from verifiers.v1.utils.install import ensure_installed -from verifiers.v1.utils.generic import generic_type from verifiers.v1.task import Task -from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.taskset import Taskset +from verifiers.v1.utils.generic import generic_type, prefix_validation_error +from verifiers.v1.utils.install import ensure_installed def builtin_harness_ids() -> list[str]: """The harness ids that ship with verifiers (the `verifiers.v1.harnesses` subpackages).""" - import verifiers.v1.harnesses as harnesses + from verifiers.v1 import harnesses return sorted(m.name for m in pkgutil.iter_modules(harnesses.__path__)) @@ -59,7 +58,7 @@ def narrow_plugin_field( if field == "harness" else "" ) - raise ValueError( + raise ValueError( # noqa: TRY004 - Pydantic validators must raise ValueError f"{field}.id needs an id, and none was given (got {ident!r}); " f"pass the id right after the flag{hint}" ) diff --git a/verifiers/v1/mcp/__init__.py b/verifiers/v1/mcp/__init__.py index f7d5db8bb1..5b98f22827 100644 --- a/verifiers/v1/mcp/__init__.py +++ b/verifiers/v1/mcp/__init__.py @@ -1,6 +1,6 @@ from verifiers.v1.mcp.launch import ( - serve, SharedToolServer, + serve, serve_shared, serve_tools, ) @@ -9,11 +9,11 @@ __all__ = [ "ServerBase", - "Toolset", + "SharedToolServer", "SharedToolsetConfig", + "Toolset", "ToolsetConfig", "serve", - "SharedToolServer", "serve_shared", "serve_tools", ] diff --git a/verifiers/v1/mcp/server.py b/verifiers/v1/mcp/server.py index 487e8e0994..5c35c6fcef 100644 --- a/verifiers/v1/mcp/server.py +++ b/verifiers/v1/mcp/server.py @@ -6,7 +6,8 @@ import inspect import logging import os -from typing import TYPE_CHECKING, Callable, ClassVar, Generic, TypeVar +from collections.abc import Callable +from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar from pydantic import TypeAdapter, ValidationError from pydantic_config import BaseConfig @@ -246,7 +247,7 @@ def _serve(self) -> None: # before setup so an expensive setup does not block port discovery. sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((host, int(os.environ.get("MCP_PORT", 0)))) + sock.bind((host, int(os.environ.get("MCP_PORT", "0")))) port_file = os.environ.get("MCP_PORT_FILE") if port_file: Path(port_file).write_text(str(sock.getsockname()[1])) diff --git a/verifiers/v1/push.py b/verifiers/v1/push.py index f16fde4745..d3122fe5f1 100644 --- a/verifiers/v1/push.py +++ b/verifiers/v1/push.py @@ -272,7 +272,7 @@ def post(path: str, body: dict) -> dict: ) resp.raise_for_status() post(f"/evaluations/{eval_id}/finalize", {"metrics": metrics}) - except Exception as e: + except Exception as e: # noqa: BLE001 - push is best-effort across the full upload logger.warning("--push: upload failed (%s: %s); skipping", type(e).__name__, e) return finish(error=f"{type(e).__name__}: {e}") diff --git a/verifiers/v1/retries.py b/verifiers/v1/retries.py index a19d1ad23e..b9f62234b0 100644 --- a/verifiers/v1/retries.py +++ b/verifiers/v1/retries.py @@ -18,7 +18,6 @@ from collections.abc import Awaitable, Callable from typing import TYPE_CHECKING -from verifiers.v1.configs.retries import RetryConfig from tenacity import ( AsyncRetrying, RetryCallState, @@ -28,6 +27,8 @@ wait_exponential_jitter, ) +from verifiers.v1.configs.retries import RetryConfig + if TYPE_CHECKING: from verifiers.v1.episode import Episode from verifiers.v1.trace import Error diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 3df8632349..8db36ccda9 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -25,9 +25,8 @@ from contextlib import AsyncExitStack, asynccontextmanager from verifiers import __version__ -from verifiers.v1.configs.agent import AgentConfig -from verifiers.v1.harness import Harness from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.agent import AgentConfig from verifiers.v1.decorators import discover_decorated, invoke from verifiers.v1.dialects import parse_message from verifiers.v1.errors import ( @@ -37,19 +36,20 @@ ToolsetError, boundary, ) +from verifiers.v1.harness import Harness from verifiers.v1.interception import ( Interception, InterceptionServer, Slot, requires_tunnel, ) -from verifiers.v1.session import RolloutLimits, RolloutSession +from verifiers.v1.mcp import SharedToolServer, serve_tools from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, make_runtime, ) -from verifiers.v1.mcp import SharedToolServer, serve_tools +from verifiers.v1.session import RolloutLimits, RolloutSession from verifiers.v1.state import state_cls from verifiers.v1.task import Task, TaskData from verifiers.v1.trace import AgentInfo, Trace, TraceTask, VersionInfo @@ -86,9 +86,8 @@ async def _serve_interception( shared_tools.values(), ) server = InterceptionServer(requires_tunnel=tunneled) - async with server: - async with server.acquire(session) as slot: - yield slot + async with server, server.acquire(session) as slot: + yield slot class RolloutRun: @@ -289,7 +288,7 @@ async def open(self) -> bool: # Setup and service provisioning are complete. Apply the runtime's # execution policy while preserving the framework routes the agent uses. await runtime.prepare_execution([self._endpoint, *self._urls.values()]) - except Exception as e: + except Exception as e: # noqa: BLE001 - setup boundary records every rollout failure self.fail(e) return False except BaseException: @@ -344,7 +343,7 @@ async def step(self, messages: Messages | None = None) -> bool: else: self.fail(e) return False - except Exception as e: + except Exception as e: # noqa: BLE001 - harness boundary records every rollout failure real = self._session.error if real is not None and isinstance(e, RolloutError): real.__cause__ = e @@ -419,7 +418,7 @@ async def close(self) -> Trace: self._scoring_timeout, ) trace.timing.scoring.end = time.time() - except Exception as e: + except Exception as e: # noqa: BLE001 - finalize boundary records every rollout failure self.fail(e) finally: trace.is_completed = True diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index f84e2b273b..fdfc7303b5 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -52,24 +52,24 @@ def runtime_is_local(config: RuntimeConfig) -> bool: __all__ = [ - "ProgramResult", - "Runtime", - "RuntimeConfig", - "RuntimeInfo", "BaseRuntimeInfo", - "NetworkPolicyConfig", - "make_runtime", - "runtime_is_local", - "SubprocessConfig", - "SubprocessRuntime", - "SubprocessRuntimeInfo", "DockerConfig", "DockerRuntime", "DockerRuntimeInfo", - "PrimeConfig", - "PrimeRuntime", - "PrimeRuntimeInfo", "ModalConfig", "ModalRuntime", "ModalRuntimeInfo", + "NetworkPolicyConfig", + "PrimeConfig", + "PrimeRuntime", + "PrimeRuntimeInfo", + "ProgramResult", + "Runtime", + "RuntimeConfig", + "RuntimeInfo", + "SubprocessConfig", + "SubprocessRuntime", + "SubprocessRuntimeInfo", + "make_runtime", + "runtime_is_local", ] diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index e8d80cb3b9..78d404a64f 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -13,6 +13,7 @@ from pathlib import PurePosixPath from typing import ClassVar, Self +from pydantic import Field from pydantic_config import BaseConfig from verifiers.v1.errors import SandboxError @@ -98,9 +99,9 @@ def cleanup_at_exit() -> None: class NetworkPolicyConfig(BaseConfig): """Shared execution-time policy surface for runtimes that support it.""" - allow: list[str] = ["*"] + allow: list[str] = Field(default_factory=lambda: ["*"]) """Destinations allowed during execution; `*` leaves egress unrestricted.""" - block: list[str] = [] + block: list[str] = Field(default_factory=list) """Destinations denied during execution.""" @property diff --git a/verifiers/v1/runtimes/docker/__init__.py b/verifiers/v1/runtimes/docker/__init__.py index bbeee85409..12d4cb8c52 100644 --- a/verifiers/v1/runtimes/docker/__init__.py +++ b/verifiers/v1/runtimes/docker/__init__.py @@ -388,4 +388,5 @@ def cleanup(self) -> None: stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30, + check=False, ) diff --git a/verifiers/v1/runtimes/docker/egress.py b/verifiers/v1/runtimes/docker/egress.py index e5956f00c1..2323d6ef8d 100644 --- a/verifiers/v1/runtimes/docker/egress.py +++ b/verifiers/v1/runtimes/docker/egress.py @@ -163,7 +163,7 @@ async def _handle( client.receive_data(head) request = client.next_event() if not isinstance(request, h11.Request): - raise ValueError("expected an HTTP request") + raise TypeError("expected an HTTP request") authorization = next( ( value @@ -327,7 +327,7 @@ async def _handle( response_started = True writer.write(chunk) await _drain(writer) - except Exception: + except Exception: # noqa: BLE001 - proxy failures become a generic 502 if not response_started: with contextlib.suppress(Exception): writer.write( diff --git a/verifiers/v1/runtimes/limiters.py b/verifiers/v1/runtimes/limiters.py index 0128900386..d078db677d 100644 --- a/verifiers/v1/runtimes/limiters.py +++ b/verifiers/v1/runtimes/limiters.py @@ -12,6 +12,7 @@ import os import tempfile import time +from typing import Self _LIMITER_DIR = os.path.join(tempfile.gettempdir(), "vf-rate-limiters") @@ -44,7 +45,7 @@ def _reserve(self) -> float: finally: fcntl.flock(f.fileno(), fcntl.LOCK_UN) - async def __aenter__(self) -> "CreationLimiter": + async def __aenter__(self) -> Self: wait = await asyncio.to_thread(self._reserve) if wait > 0: await asyncio.sleep(wait) diff --git a/verifiers/v1/runtimes/modal.py b/verifiers/v1/runtimes/modal.py index 14f5a67b8f..7c65a7331b 100644 --- a/verifiers/v1/runtimes/modal.py +++ b/verifiers/v1/runtimes/modal.py @@ -203,6 +203,6 @@ async def teardown(self) -> None: return try: await sandbox.terminate.aio() - except Exception as e: + except Exception as e: # noqa: BLE001 - provider teardown is best-effort logger.warning("modal: failed to terminate sandbox %s: %s", self.info.id, e) self._sandbox = None diff --git a/verifiers/v1/runtimes/prime.py b/verifiers/v1/runtimes/prime.py index b8de3386b7..f28364e13f 100644 --- a/verifiers/v1/runtimes/prime.py +++ b/verifiers/v1/runtimes/prime.py @@ -16,7 +16,7 @@ from typing import ClassVar, Literal from urllib.parse import urlsplit -from pydantic import model_validator +from pydantic import Field, model_validator from verifiers.v1.errors import SandboxError from verifiers.v1.runtimes.base import ( @@ -49,7 +49,7 @@ class PrimeConfig(NetworkPolicyConfig): """Request guaranteed (vs best-effort) capacity.""" region: str | None = None """Region to provision in (None = provider-chosen).""" - labels: list[str] = [] + labels: list[str] = Field(default_factory=list) """Labels attached to the sandbox.""" # TaskData.resources uses these units; non-default runtime config values take precedence. cpu: float = 1.0 @@ -332,7 +332,7 @@ async def teardown(self) -> None: if self.info.id is not None: # keep info.id available after teardown try: await client.delete(self.info.id) - except Exception as e: + except Exception as e: # noqa: BLE001 - provider teardown is best-effort logger.warning( "prime: failed to delete sandbox %s: %s", self.info.id, e ) diff --git a/verifiers/v1/runtimes/subprocess.py b/verifiers/v1/runtimes/subprocess.py index e4a74deae3..baa35454db 100644 --- a/verifiers/v1/runtimes/subprocess.py +++ b/verifiers/v1/runtimes/subprocess.py @@ -6,7 +6,7 @@ import shutil import signal from pathlib import Path -from typing import Literal +from typing import ClassVar, Literal from pydantic_config import BaseConfig @@ -30,8 +30,8 @@ class SubprocessRuntimeInfo(SubprocessConfig, BaseRuntimeInfo): class SubprocessRuntime(Runtime): # Share prepared script environments across the worker's per-rollout runtimes. - _interpreters: dict[str, str] = {} - _locks: dict[str, asyncio.Lock] = {} + _interpreters: ClassVar[dict[str, str]] = {} + _locks: ClassVar[dict[str, asyncio.Lock]] = {} def __init__(self, config: SubprocessConfig, name: str | None = None) -> None: super().__init__(name) diff --git a/verifiers/v1/scoring.py b/verifiers/v1/scoring.py index 5406a87cd5..f3a9087670 100644 --- a/verifiers/v1/scoring.py +++ b/verifiers/v1/scoring.py @@ -101,7 +101,7 @@ def verify_boxed_math_answer( timeout_seconds=timeout_seconds, ) ) - except (Exception, MathVerifyTimeout): + except (Exception, MathVerifyTimeout): # noqa: BLE001 - malformed answers score zero return 0.0 diff --git a/verifiers/v1/serve/__init__.py b/verifiers/v1/serve/__init__.py index fdfce74e8e..85940ef270 100644 --- a/verifiers/v1/serve/__init__.py +++ b/verifiers/v1/serve/__init__.py @@ -13,17 +13,17 @@ ) __all__ = [ + "EnvClient", "EnvServer", "EnvServerPool", - "serve_env", - "env_config_data", - "EnvClient", "HealthRequest", "HealthResponse", "InfoRequest", "InfoResponse", - "RunRequest", - "RunResponse", "RunGroupRequest", "RunGroupResponse", + "RunRequest", + "RunResponse", + "env_config_data", + "serve_env", ] diff --git a/verifiers/v1/serve/client.py b/verifiers/v1/serve/client.py index 41ff86cc07..385ab35bd1 100644 --- a/verifiers/v1/serve/client.py +++ b/verifiers/v1/serve/client.py @@ -20,6 +20,7 @@ import zmq.asyncio from verifiers.v1.clients.config import ClientConfig +from verifiers.v1.episode import WireEpisode from verifiers.v1.serve.types import ( BaseRequest, BaseResponse, @@ -32,7 +33,6 @@ RunRequest, RunResponse, ) -from verifiers.v1.episode import WireEpisode from verifiers.v1.trace import WireTrace from verifiers.v1.types import SamplingConfig @@ -86,7 +86,7 @@ async def _request( ) try: data = await asyncio.wait_for(future, timeout) - except (asyncio.TimeoutError, asyncio.CancelledError): + except (TimeoutError, asyncio.CancelledError): self._pending.pop(request_id, None) raise if response_type in (HealthResponse, InfoResponse): @@ -119,7 +119,7 @@ async def health(self, timeout: float = 2.0) -> bool: return ( await self._request(HealthRequest(), HealthResponse, timeout=timeout) ).success - except asyncio.TimeoutError: + except TimeoutError: return False async def wait_for_server_startup( diff --git a/verifiers/v1/serve/types.py b/verifiers/v1/serve/types.py index 90981a41b7..8aaa68eff6 100644 --- a/verifiers/v1/serve/types.py +++ b/verifiers/v1/serve/types.py @@ -3,8 +3,8 @@ from pydantic import BaseModel, Field, field_serializer, model_validator from verifiers.v1.clients.config import ClientConfig -from verifiers.v1.task import WireTaskData # noqa: F401 (docstring reference) from verifiers.v1.episode import WireEpisode +from verifiers.v1.task import WireTaskData # noqa: F401 (docstring reference) from verifiers.v1.trace import WireTrace from verifiers.v1.types import SamplingConfig diff --git a/verifiers/v1/task.py b/verifiers/v1/task.py index f3c819741a..9b59119c6d 100644 --- a/verifiers/v1/task.py +++ b/verifiers/v1/task.py @@ -29,13 +29,13 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, ClassVar, Generic -from pydantic import ConfigDict +from pydantic import ConfigDict, Field from pydantic_config import BaseConfig from typing_extensions import TypeVar +from verifiers.v1.configs.task import TaskConfig from verifiers.v1.decorators import discover_decorated, invoke_all from verifiers.v1.errors import TaskError, boundary -from verifiers.v1.configs.task import TaskConfig from verifiers.v1.state import StateT from verifiers.v1.types import Messages, StrictBaseModel, content_text from verifiers.v1.utils.generic import generic_type @@ -114,11 +114,11 @@ class TaskData(StrictBaseModel): system_prompt: str | None = None image: str | None = None workdir: str | None = None - network_allow: list[str] = ["*"] + network_allow: list[str] = Field(default_factory=lambda: ["*"]) """Execution-time destinations requested by this task. `*` leaves the runtime allowlist unchanged; a concrete list replaces a wildcard or combines with existing entries. Prime runtimes accept host-level entries and require `vm=true`.""" - network_block: list[str] = [] + network_block: list[str] = Field(default_factory=list) """Execution-time destinations denied by this task and combined with runtime blocks. Prime runtimes cannot combine an allowlist and a blocklist.""" timeout: TaskTimeout = TaskTimeout() diff --git a/verifiers/v1/tasksets/harbor/taskset.py b/verifiers/v1/tasksets/harbor/taskset.py index e295ed8f64..efbd34da8a 100644 --- a/verifiers/v1/tasksets/harbor/taskset.py +++ b/verifiers/v1/tasksets/harbor/taskset.py @@ -24,11 +24,11 @@ from pydantic import Field +from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.decorators import reward from verifiers.v1.errors import SandboxError from verifiers.v1.runtimes import Runtime from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout -from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.taskset import Taskset from verifiers.v1.types import StrictBaseModel @@ -83,14 +83,14 @@ class HarborData(TaskData): name, and description. The remaining fields mirror Harbor metadata. """ - keywords: list[str] = [] - authors: list[Author] = [] + keywords: list[str] = Field(default_factory=list) + authors: list[Author] = Field(default_factory=list) difficulty: str | None = None category: str | None = None - tags: list[str] = [] + tags: list[str] = Field(default_factory=list) task_dir: str = "" """Host path to the task dir; used to stage tests/ to verify.""" - verifier_env: dict[str, str] = {} + verifier_env: dict[str, str] = Field(default_factory=dict) """Raw [verifier.env] entries (literals or `${VAR}`/`${VAR:-default}` templates). Resolved against the host environment at scoring time, like `harbor run` — so a verifier that needs judge API keys or configuration actually receives them.""" @@ -243,7 +243,7 @@ def resolve_image( return None -def size_to_mb(size: str | int | float) -> float: +def size_to_mb(size: str | float) -> float: """A Harbor size in MB, from either schema: current integer-MB fields or the legacy schema-1.0 size strings ("8G", "512M", "64K").""" if not isinstance(size, str): @@ -275,7 +275,8 @@ def parse_resources(env: dict, multiplier: float = 1.0) -> TaskResources: def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborData: # Harbor is optional, so importing its schema is deferred until a Harbor task loads. - from harbor.models.task.config import NetworkMode, TaskConfig as HarborTaskConfig + from harbor.models.task.config import NetworkMode + from harbor.models.task.config import TaskConfig as HarborTaskConfig config = tomllib.loads((task_dir / "task.toml").read_text()) parsed = HarborTaskConfig.model_validate(config) diff --git a/verifiers/v1/tasksets/lean/__init__.py b/verifiers/v1/tasksets/lean/__init__.py index 1ab0923828..6a3ee74438 100644 --- a/verifiers/v1/tasksets/lean/__init__.py +++ b/verifiers/v1/tasksets/lean/__init__.py @@ -10,9 +10,9 @@ LEAN_PROJECT_PATH, PROOF_FILE_PATH, LeanConfig, - LeanTaskConfig, LeanDatasetConfig, LeanTask, + LeanTaskConfig, LeanTaskset, ) @@ -21,9 +21,9 @@ "LEAN_PROJECT_PATH", "PROOF_FILE_PATH", "LeanConfig", - "LeanTaskConfig", "LeanDatasetConfig", "LeanTask", + "LeanTaskConfig", "LeanTaskset", "build_starter_file", "expected_protected_signature", diff --git a/verifiers/v1/tasksets/lean/taskset.py b/verifiers/v1/tasksets/lean/taskset.py index b62c146327..58ccdee54e 100644 --- a/verifiers/v1/tasksets/lean/taskset.py +++ b/verifiers/v1/tasksets/lean/taskset.py @@ -14,12 +14,12 @@ from pydantic_config import BaseConfig +from verifiers.v1.configs.task import TaskConfig +from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.decorators import reward from verifiers.v1.runtimes import Runtime from verifiers.v1.state import State -from verifiers.v1.configs.task import TaskConfig from verifiers.v1.task import Task, TaskData, TaskResources -from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.taskset import Taskset from verifiers.v1.tasksets.lean.scoring import ( build_starter_file, @@ -222,4 +222,4 @@ def _build_prompt(self, formal_statement: str, header: str) -> str: return block -__all__ = ["LeanTask", "LeanConfig", "LeanTaskset"] +__all__ = ["LeanConfig", "LeanTask", "LeanTaskset"] diff --git a/verifiers/v1/tasksets/openenv/taskset.py b/verifiers/v1/tasksets/openenv/taskset.py index 07f56331a2..ea7d3f5378 100644 --- a/verifiers/v1/tasksets/openenv/taskset.py +++ b/verifiers/v1/tasksets/openenv/taskset.py @@ -11,7 +11,7 @@ from typing import Any, Self import httpx -from pydantic import model_validator +from pydantic import Field, model_validator import verifiers.v1 as vf @@ -32,9 +32,9 @@ class OpenEnvConfig(vf.TasksetConfig): use_docker: bool = False """Use OpenEnv's Docker provider instead of the default UV provider. The engine runs host-side (in the eval process), so this needs Docker on the host.""" - provider_kwargs: dict[str, Any] = {} + provider_kwargs: dict[str, Any] = Field(default_factory=dict) """Extra arguments for OpenEnv's provider (`GenericEnvClient.from_env`).""" - resets: list[dict[str, Any]] = [{}] + resets: list[dict[str, Any]] = Field(default_factory=lambda: [{}]) """One finite task per set of arguments passed to OpenEnv's `reset`.""" @model_validator(mode="after") diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index ae90009e99..235f56f3dc 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -7,19 +7,18 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Annotated, Any, Generic, Literal -from typing_extensions import TypeVar - import numpy as np from pydantic import Field, PrivateAttr from renderers.base import MultiModalData +from typing_extensions import TypeVar if TYPE_CHECKING: from verifiers.v1.judge import JudgeResponse from verifiers.v1 import graph +from verifiers.v1.configs.agent import AgentConfig, WireAgentConfig from verifiers.v1.errors import ProviderError from verifiers.v1.graph import MessageNode -from verifiers.v1.configs.agent import AgentConfig, WireAgentConfig from verifiers.v1.runtimes import RuntimeInfo from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData diff --git a/verifiers/v1/utils/aio.py b/verifiers/v1/utils/aio.py index a62887f509..ab14b290e0 100644 --- a/verifiers/v1/utils/aio.py +++ b/verifiers/v1/utils/aio.py @@ -24,7 +24,7 @@ async def run_shielded(coro: Awaitable[T]) -> T: await asyncio.shield(task) except asyncio.CancelledError as e: cancelled = e - except BaseException: + except BaseException: # noqa: BLE001, S110 - re-raised or chained below pass # `coro` itself failed → task is done; re-raised (or chained) below if cancelled is not None: raise cancelled from (None if task.cancelled() else task.exception()) diff --git a/verifiers/v1/utils/compile.py b/verifiers/v1/utils/compile.py index faf2d38048..2153e283ef 100644 --- a/verifiers/v1/utils/compile.py +++ b/verifiers/v1/utils/compile.py @@ -38,7 +38,7 @@ def resolve_runtime_config( if ( task.data.workdir is not None and workdir_spec is not None - and getattr(config, "workdir") == workdir_spec.default + and config.workdir == workdir_spec.default ): updates["workdir"] = task.data.workdir task_network_policy = "*" not in task.data.network_allow or bool( diff --git a/verifiers/v1/utils/version.py b/verifiers/v1/utils/version.py index 67d4060cca..f7f15171f5 100644 --- a/verifiers/v1/utils/version.py +++ b/verifiers/v1/utils/version.py @@ -27,6 +27,7 @@ def verifiers_commit() -> str | None: capture_output=True, text=True, timeout=5, + check=False, ) except (OSError, subprocess.TimeoutExpired): return None From 695ac324586f83b9e879b63b226fa21c2d6211bd Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:32:27 +0200 Subject: [PATCH 2/4] Update Ruff and ty tooling --- .pre-commit-config.yaml | 6 +- pyproject.toml | 7 +- uv.lock | 221 +++++++++--------- .../clients/nemorl_chat_completions_client.py | 6 +- .../clients/openai_chat_completions_client.py | 2 +- .../openai_chat_completions_token_client.py | 4 +- verifiers/envs/env_group.py | 12 +- verifiers/envs/environment.py | 6 +- .../envs/experimental/composable/task.py | 6 +- .../composable/tasksets/cp/test_utils.py | 2 +- verifiers/envs/experimental/harbor_env/mcp.py | 12 +- .../browser_env/modes/cua_mode.py | 24 +- verifiers/envs/integrations/openenv_env.py | 2 +- verifiers/envs/integrations/textarena_env.py | 2 +- verifiers/envs/tool_env.py | 2 +- verifiers/gepa/adapter.py | 4 +- .../experimental/hybrid_math_rubric.py | 2 +- verifiers/rubrics/rubric.py | 16 +- verifiers/scripts/gepa.py | 2 +- verifiers/types.py | 4 +- verifiers/utils/async_utils.py | 4 +- verifiers/utils/eval_utils.py | 8 +- verifiers/utils/interception_utils.py | 2 +- verifiers/utils/message_utils.py | 10 +- verifiers/utils/save_utils.py | 4 +- 25 files changed, 188 insertions(+), 182 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 866e035306..ebe0048501 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,19 +9,19 @@ repos: hooks: - id: ruff-check name: ruff check - entry: uv run ruff check --fix + entry: uv run --locked ruff check --fix language: system types_or: [python, pyi] exclude: ^environments/.*/tasks/ - id: ruff-format name: ruff format - entry: uv run ruff format + entry: uv run --locked ruff format language: system types_or: [python, pyi] exclude: ^environments/.*/tasks/ - id: ty name: ty (ci parity) - entry: uv run --python 3.13 ty check verifiers + entry: uv run --locked --python 3.13 ty check verifiers language: system pass_filenames: false stages: [pre-push] diff --git a/pyproject.toml b/pyproject.toml index 7748ba1599..5aee69a11b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,9 +57,9 @@ dependencies = [ [dependency-groups] dev = [ - "ruff>=0.9.0", + "ruff==0.16.0", "pre-commit>=2.19.0", - "ty>=0.0.1a29,<0.0.22", + "ty==0.0.63", "pytest>=7.0.0,<9.1.0", "pytest-asyncio>=0.21.0", "pytest-cov>=4.0.0", @@ -135,6 +135,9 @@ scratchpad-v1 = { path = "environments/scratchpad_v1", editable = true } kuhn-poker-v1 = { path = "environments/kuhn_poker_v1", editable = true } [tool.uv.exclude-newer-package] +# Bounded cutoffs for the explicitly requested tool upgrades. +ruff = "2026-07-27" +ty = "2026-07-27" # PrimeIntellect-published on PyPI (trusted publisher) prime-tunnel = false prime-sandboxes = false diff --git a/uv.lock b/uv.lock index e09027976a..f0dc98ecb5 100644 --- a/uv.lock +++ b/uv.lock @@ -21,7 +21,9 @@ exclude-newer-span = "P7D" prime-tunnel = false prime-sandboxes = false prime-pydantic-config = false +ty = "2026-07-27T22:00:00Z" renderers = false +ruff = "2026-07-27T22:00:00Z" [[package]] name = "aiofile" @@ -681,9 +683,9 @@ name = "claude-agent-sdk" version = "0.2.116" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio", marker = "python_full_version >= '3.12'" }, - { name = "mcp", marker = "python_full_version >= '3.12'" }, - { name = "sniffio", marker = "python_full_version >= '3.12'" }, + { name = "anyio" }, + { name = "mcp" }, + { name = "sniffio" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/b0f01a18bb119e595cc1e66d4adbe39bb09879a570e54793c222a142b662/claude_agent_sdk-0.2.116.tar.gz", hash = "sha256:37d6c9d98960c7ea41d1a7fd3331ad0b1bef68e559ed9cc5ccf9d00ae68adfcd", size = 268648, upload-time = "2026-07-11T01:04:47.192Z" } wheels = [ @@ -1003,7 +1005,7 @@ name = "deprecation" version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "packaging", marker = "python_full_version >= '3.12'" }, + { name = "packaging" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } wheels = [ @@ -1024,7 +1026,7 @@ name = "dirhash" version = "0.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "scantree", marker = "python_full_version >= '3.12'" }, + { name = "scantree" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1d/70/49f93897f3a4f7ab5f20a854ebc91aad47854e9fb2cd169e3a4452fa3f5e/dirhash-0.5.0.tar.gz", hash = "sha256:e60760f0ab2e935d8cb088923ea2c6492398dca42cec785df778985fd4cd5386", size = 21377, upload-time = "2024-08-03T22:14:13.322Z" } wheels = [ @@ -1573,26 +1575,26 @@ name = "harbor" version = "0.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "claude-agent-sdk", marker = "python_full_version >= '3.12'" }, - { name = "datasets", marker = "python_full_version >= '3.12'" }, - { name = "dirhash", marker = "python_full_version >= '3.12'" }, - { name = "fastapi", marker = "python_full_version >= '3.12'" }, - { name = "httpx", marker = "python_full_version >= '3.12'" }, - { name = "jinja2", marker = "python_full_version >= '3.12'" }, - { name = "litellm", marker = "python_full_version >= '3.12'" }, - { name = "packaging", marker = "python_full_version >= '3.12'" }, - { name = "pathspec", marker = "python_full_version >= '3.12'" }, - { name = "pydantic", marker = "python_full_version >= '3.12'" }, - { name = "python-dotenv", marker = "python_full_version >= '3.12'" }, - { name = "pyyaml", marker = "python_full_version >= '3.12'" }, - { name = "requests", marker = "python_full_version >= '3.12'" }, - { name = "rich", marker = "python_full_version >= '3.12'" }, - { name = "shortuuid", marker = "python_full_version >= '3.12'" }, - { name = "supabase", marker = "python_full_version >= '3.12'" }, - { name = "tenacity", marker = "python_full_version >= '3.12'" }, - { name = "toml", marker = "python_full_version >= '3.12'" }, - { name = "typer", marker = "python_full_version >= '3.12'" }, - { name = "uvicorn", marker = "python_full_version >= '3.12'" }, + { name = "claude-agent-sdk" }, + { name = "datasets" }, + { name = "dirhash" }, + { name = "fastapi" }, + { name = "httpx" }, + { name = "jinja2" }, + { name = "litellm" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "rich" }, + { name = "shortuuid" }, + { name = "supabase" }, + { name = "tenacity" }, + { name = "toml" }, + { name = "typer" }, + { name = "uvicorn" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/f3/cba38a11b0cca01ecd471437869f1cd57d86a3b1e4091b8fd3d112cba8af/harbor-0.14.0.tar.gz", hash = "sha256:0ffffc25a618e4b7bf2b901a8c717c8bc28877f5aa8c37be47f1ec8c301cb0b6", size = 1241271, upload-time = "2026-06-17T16:43:23.495Z" } wheels = [ @@ -1704,7 +1706,7 @@ wheels = [ [package.optional-dependencies] http2 = [ - { name = "h2", marker = "python_full_version >= '3.12'" }, + { name = "h2" }, ] [[package]] @@ -2124,18 +2126,18 @@ name = "litellm" version = "1.91.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp", marker = "python_full_version >= '3.12'" }, - { name = "click", marker = "python_full_version >= '3.12'" }, - { name = "fastuuid", marker = "python_full_version >= '3.12'" }, - { name = "httpx", marker = "python_full_version >= '3.12'" }, - { name = "importlib-metadata", marker = "python_full_version >= '3.12'" }, - { name = "jinja2", marker = "python_full_version >= '3.12'" }, - { name = "jsonschema", marker = "python_full_version >= '3.12'" }, - { name = "openai", marker = "python_full_version >= '3.12'" }, - { name = "pydantic", marker = "python_full_version >= '3.12'" }, - { name = "python-dotenv", marker = "python_full_version >= '3.12'" }, - { name = "tiktoken", marker = "python_full_version >= '3.12'" }, - { name = "tokenizers", marker = "python_full_version >= '3.12'" }, + { name = "aiohttp" }, + { name = "click" }, + { name = "fastuuid" }, + { name = "httpx" }, + { name = "importlib-metadata" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "openai" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "tiktoken" }, + { name = "tokenizers" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6b/5d/2dfda0e057511ba24904c5c9af720512b1fc86893821947fa52f4cc0231b/litellm-1.91.2.tar.gz", hash = "sha256:e9aa292b95f25fa9d127e47a6e7266a2a99f7a1645f41100a411d7a8a77a6a46", size = 14872927, upload-time = "2026-07-11T06:04:18.847Z" } wheels = [ @@ -3139,10 +3141,10 @@ name = "postgrest" version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "deprecation", marker = "python_full_version >= '3.12'" }, - { name = "httpx", extra = ["http2"], marker = "python_full_version >= '3.12'" }, - { name = "pydantic", marker = "python_full_version >= '3.12'" }, - { name = "yarl", marker = "python_full_version >= '3.12'" }, + { name = "deprecation" }, + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "yarl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e2/22/88c470d8838d2678a44e0172d061630b8837cba3fb7fb492e28f6578c309/postgrest-2.31.0.tar.gz", hash = "sha256:2f395d84b2ee34fc57622ff2f711df603e2ede625f98e5015240741888f7bd0c", size = 14419, upload-time = "2026-06-04T13:37:20.474Z" } wheels = [ @@ -3931,9 +3933,9 @@ name = "realtime" version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic", marker = "python_full_version >= '3.12'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, - { name = "websockets", marker = "python_full_version >= '3.12'" }, + { name = "pydantic" }, + { name = "typing-extensions" }, + { name = "websockets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/34/54a1eaaefa24db5cb12596fd74792e08efa53ed30dc5bce2c0a68ded6146/realtime-2.31.0.tar.gz", hash = "sha256:9e641cb4d77ca0fe768515f8cf9f83550c79f49ce1550a95afc2dc0e252be8c9", size = 18716, upload-time = "2026-06-04T13:37:22.089Z" } wheels = [ @@ -4199,27 +4201,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.15.21" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/36/6f65aa9989acdec45d417192d8f4e7921931d8a6cf87ac74bce3eed98a8e/ruff-0.15.21.tar.gz", hash = "sha256:d0cfc841c572283c36548f82664a54ce6565567f1b0d5b4cf2caac693d8b7500", size = 4769401, upload-time = "2026-07-09T20:01:34.005Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/c6/ede15cac6839f3dbce52565c8f5164a8210e669c7bc4decb03e5bdf47d0d/ruff-0.15.21-py3-none-linux_armv6l.whl", hash = "sha256:63ea0e965e5d73c90e95b2434beeafc70820536717f561b32ab6e777cb9bdf5d", size = 10854342, upload-time = "2026-07-09T20:00:53.998Z" }, - { url = "https://files.pythonhosted.org/packages/28/9d/d825b07ee7ea9e2d61df92a860033c94e06e7300d50a1c2653aac27d24fe/ruff-0.15.21-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0f212c5d7d54c01bbfe6dcab02b724a39300f3e34ed7acbe995ccb320a2c58bd", size = 11139539, upload-time = "2026-07-09T20:00:57.809Z" }, - { url = "https://files.pythonhosted.org/packages/f5/de/3b107712e642f063c7a9e0887c427b22cb44097de5aab36c05f2e280670c/ruff-0.15.21-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e6312e41bc96791299614995ea3a977c5857c3b5662b1ecef6755b02b87cb646", size = 10595437, upload-time = "2026-07-09T20:01:00.006Z" }, - { url = "https://files.pythonhosted.org/packages/9a/6f/b4523cc90ba239ede441447a19d0c968846a3012e5a0b0c5b62831a3d5e3/ruff-0.15.21-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01d65b4831c6b2a4ba8ee6faa84049d44d982b7a706e622c4094c509e51673be", size = 10990053, upload-time = "2026-07-09T20:01:02.187Z" }, - { url = "https://files.pythonhosted.org/packages/92/cc/c6a9872a5375f0628875481cf2f66b13d7d865bf3ca2e57f91c7e762d976/ruff-0.15.21-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2c5a913a589120ce67933d5d05fd6ddbcc2481c6a054980ee767f7414c72b4fd", size = 10666096, upload-time = "2026-07-09T20:01:04.299Z" }, - { url = "https://files.pythonhosted.org/packages/ab/97/c621f7a17e097f1790fa3af6374138823b330b2d03fc38337945daca212c/ruff-0.15.21-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef04b681d02ad4dc9620f00f83ac5c22f652d0e9a9cfe431d219b16ad5ccc41", size = 11537011, upload-time = "2026-07-09T20:01:06.771Z" }, - { url = "https://files.pythonhosted.org/packages/ea/51/d928727e476e25ccc57c6f449ffd80241a651a973ad949d39cfb2a771d28/ruff-0.15.21-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16d090c0740916594157e75b80d666eab8e78083b39b3b0e1d698f4670a17b86", size = 12347101, upload-time = "2026-07-09T20:01:08.859Z" }, - { url = "https://files.pythonhosted.org/packages/1e/88/8cd62026802b16018ad06931d87997cf795ba2a6239ab659606c87d96bf0/ruff-0.15.21-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a10e74757dd65004d779b73e2f3c5210156d9980b41224d50d2ebcf1db51e67", size = 11572001, upload-time = "2026-07-09T20:01:11.092Z" }, - { url = "https://files.pythonhosted.org/packages/b2/97/f63084cf55444fc110e8cb985ebfcc592af47f597d44453d778cb81bc156/ruff-0.15.21-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bab0905d2f29e0d9fbc3c373ed23db0095edaa3f71f1f4f519ec15134d9e85c8", size = 11549239, upload-time = "2026-07-09T20:01:13.27Z" }, - { url = "https://files.pythonhosted.org/packages/9d/77/f107da4a2874b7715914b03f09ba9c54424de3ff8a1cc5d015d3ee2ce0ac/ruff-0.15.21-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:00eca240af5789fec6fe7df74c088cc1f9644ed83027113468efba7c92b94075", size = 11535340, upload-time = "2026-07-09T20:01:15.206Z" }, - { url = "https://files.pythonhosted.org/packages/d5/e9/601deb322d3303a7bf212b0100ead6f2ee3f6a044d89c30f2f92bf83c731/ruff-0.15.21-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:262ab31557a75141325e32d3357f3597645a7f084e732b6b054dde428ecd9341", size = 10964048, upload-time = "2026-07-09T20:01:17.723Z" }, - { url = "https://files.pythonhosted.org/packages/ea/2e/0f2176d1e99c15192caea19c8c3a0a955246b4cb4de795042eeb616345cd/ruff-0.15.21-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:659c4e7a4212f83306045ec7c5e5a356d16d9a6ef4ae0c7a4d872914fc655d9d", size = 10667055, upload-time = "2026-07-09T20:01:19.73Z" }, - { url = "https://files.pythonhosted.org/packages/48/60/abd74a02e0c4214f12a68becfd30af7165cfdcb0e661ecdc60bbb949c09a/ruff-0.15.21-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9e866eab611a5f959d36df2d10e446973a3610bc42b0c15b31dc27977d59c233", size = 11242043, upload-time = "2026-07-09T20:01:21.947Z" }, - { url = "https://files.pythonhosted.org/packages/b2/c6/583075d8ccabb4b229345edcaf1545eb3d8d6be90f686a479d7e94088bbf/ruff-0.15.21-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e89bc93c0d3803ba870b55c29671bad9dc6d94bb1eb181b056b52eb05b52854f", size = 11648064, upload-time = "2026-07-09T20:01:24.023Z" }, - { url = "https://files.pythonhosted.org/packages/3a/3c/37d0ecb729a7cc2d393ea7dce316fc585680f35d93b8d62139d7d0a3700c/ruff-0.15.21-py3-none-win32.whl", hash = "sha256:01f8d5be84823c172b389e123174f781f9daf86d6c58719d603f941932195cdd", size = 10896555, upload-time = "2026-07-09T20:01:26.941Z" }, - { url = "https://files.pythonhosted.org/packages/c0/b8/e43466b2a6067ce91e669068f6e28d6c719a920f014b070d5c8731725de3/ruff-0.15.21-py3-none-win_amd64.whl", hash = "sha256:d4b8d9a2f0f12b816b50447f6eccb9f4bb01a6b82c86b50fb3b5354b458dc6d3", size = 12038772, upload-time = "2026-07-09T20:01:29.497Z" }, - { url = "https://files.pythonhosted.org/packages/dd/75/e90ab9aeece218a9fc5a5bc3ec97d0ee6bb3c4ff95869463c1de58e29a1c/ruff-0.15.21-py3-none-win_arm64.whl", hash = "sha256:6e83115d4b9377c1cbc13abf0e051f069fab0ef815ea0504a8a008cee24dd0a8", size = 11375265, upload-time = "2026-07-09T20:01:31.772Z" }, +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/94/1e5e4967626faf12fa56999cd6222dff6992ceb086ad7945756baf70c7a7/ruff-0.16.0.tar.gz", hash = "sha256:e460aafd5495ec89efaa6ced2e4a9a581116451e1c88b9d37ef497e0f8e93982", size = 4790557, upload-time = "2026-07-23T19:11:30.981Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/81/1c8818fee7ce1a04cd7d1b3172e0a8f8e4f1dc4feb7fc390e16daa8af323/ruff-0.16.0-py3-none-linux_armv6l.whl", hash = "sha256:e5115729eb08c585e5121978ba5d5b60caeae394ce21b9fb5e6cd33a1c6c9b1e", size = 10754633, upload-time = "2026-07-23T19:10:46.415Z" }, + { url = "https://files.pythonhosted.org/packages/23/df/beaf59c09d68db84304d555f188b276a77132a5d5b0b67a5c762aa143628/ruff-0.16.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3c954b1d580bfa035b41654f7858cc7e71d5fc3ac5b723dd62bd9133830ed522", size = 10969164, upload-time = "2026-07-23T19:10:50.271Z" }, + { url = "https://files.pythonhosted.org/packages/42/ce/741cd197496a1abbf51352710fd15ed995d2a2be87189c1da26a450d6e83/ruff-0.16.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e01c21d10eb1b29f47b7454e1f4056db9a3f0260c646aa88457c610291db9f81", size = 10488846, upload-time = "2026-07-23T19:10:52.639Z" }, + { url = "https://files.pythonhosted.org/packages/52/2a/a2db8e88cade358f5cdcb05674a917751074109315d014eb6352d9a893f7/ruff-0.16.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e364e5ed22ed8dc05082fd78e35308618260907ac2d3c1d637b2e682415b6c9", size = 10889729, upload-time = "2026-07-23T19:10:54.89Z" }, + { url = "https://files.pythonhosted.org/packages/42/65/62a771694ebd63029dc953e27dbad40e1588bd4860ff9fe881018fddaa49/ruff-0.16.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d327b8fc113a1d4421a04f3839d3752057c8dd1ee320223a6f3f52d04ada462a", size = 10568275, upload-time = "2026-07-23T19:10:56.993Z" }, + { url = "https://files.pythonhosted.org/packages/3f/e2/ced249fe8af5f086c5c58cc21cc3356d50f32f7401c5df87050c999620a7/ruff-0.16.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9b50c55e263103586b3dcf5f73d479eb8cb5fdb6098fec59a62891dab653717", size = 11385112, upload-time = "2026-07-23T19:10:59.615Z" }, + { url = "https://files.pythonhosted.org/packages/87/0b/05154977a8fd69eeb6c103271f55403bfd8711f5c0f8ed07489d95a504e7/ruff-0.16.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ff4a79ce3ec0172f3241943835de1c4cb4e2dcd07f0f8c2d02603dbbbee4b17", size = 12207008, upload-time = "2026-07-23T19:11:02.154Z" }, + { url = "https://files.pythonhosted.org/packages/fb/29/98225831a3a1eab0e02f4acc6ca6559a98611dcc68b6965ff4b7234627c1/ruff-0.16.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e95c448fca1fb2a18372a9440926c5a6ee789639bb975c72e7ae6d0b04218ab4", size = 11650842, upload-time = "2026-07-23T19:11:04.557Z" }, + { url = "https://files.pythonhosted.org/packages/91/66/6bd3cf90500653d55dc0ffc8507aa8300bd49d0214b2e8cb4d3fef2943ba/ruff-0.16.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f11a8d11010301d0a398a2fdef67691feca7294da6aef55e2150e8fa2cd520b", size = 11400718, upload-time = "2026-07-23T19:11:09.233Z" }, + { url = "https://files.pythonhosted.org/packages/8e/a2/a54eb4eae05d66364050a5d3b8a9c5ef88196531b3cbe7109d873f87f819/ruff-0.16.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:48044c678e9cb8698246c99b14aaccfa6601dea7379eb48a6f8f73f7a6d86cd0", size = 11426177, upload-time = "2026-07-23T19:11:11.994Z" }, + { url = "https://files.pythonhosted.org/packages/1a/be/16e3eea4b2a478a496919f5e36f17c4559e54620bd3bbac5d6affa068006/ruff-0.16.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:7aa0959bad8eb8bef50340154fc9b58678dae31fa4293afa38b44b6e552c0213", size = 10856126, upload-time = "2026-07-23T19:11:14.221Z" }, + { url = "https://files.pythonhosted.org/packages/a2/84/252eb8b868a16eec7257c14f504f77537e734b2d69c762e639e588e304a3/ruff-0.16.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:28ea2b7df8ebf7f9da6b7d47b230ab48f387c0a29be3b474c4d0740e197bb9af", size = 10571208, upload-time = "2026-07-23T19:11:16.378Z" }, + { url = "https://files.pythonhosted.org/packages/21/09/817a482f542f7570cbb4554b26e896610c7114f539b1d9e2d2145bf6bef6/ruff-0.16.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:33a3dfac8c35f81498dea9181bccc2f4c4bc8f1521a1dd9406e77643e0f0fb09", size = 11063329, upload-time = "2026-07-23T19:11:19.173Z" }, + { url = "https://files.pythonhosted.org/packages/2e/23/9403c180ca1cb9b1f7335f5c3e5305c09d49ea5b345196682a36028bde4a/ruff-0.16.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a5237a0bda500d30d81b8e07a6973a5cbc772864cbf746ae2f4e8a2e01c9f4ed", size = 11489751, upload-time = "2026-07-23T19:11:21.74Z" }, + { url = "https://files.pythonhosted.org/packages/b2/1d/1b2ef7bcde851c78d7f17f1cca13fd6dc695fc4b3d6197941e72cae5b132/ruff-0.16.0-py3-none-win32.whl", hash = "sha256:7fab76fa065c873f41ff744347c6e77bcc3dfec4bcc754dc26b63d23c0f7f5fb", size = 10785885, upload-time = "2026-07-23T19:11:23.947Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a3/d5e4ef7a56be3f928ffb90b94c25ba7d3cb9c7fe0736aeaaedf361770712/ruff-0.16.0-py3-none-win_amd64.whl", hash = "sha256:429c117f022bf481fabd9d551e7a3952b24c65e6ef44337ea09d90bebef14472", size = 11923141, upload-time = "2026-07-23T19:11:26.409Z" }, + { url = "https://files.pythonhosted.org/packages/cb/9a/8415f2657cbe200f41a4531ccededf135505a92d4a012229121f885b26f9/ruff-0.16.0-py3-none-win_arm64.whl", hash = "sha256:14296fedcd2705c77ab8235439278bbb38f285cf7da5528b00b3e330c3d4872d", size = 11273407, upload-time = "2026-07-23T19:11:28.705Z" }, ] [[package]] @@ -4263,8 +4265,8 @@ name = "scantree" version = "0.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "attrs", marker = "python_full_version >= '3.12'" }, - { name = "pathspec", marker = "python_full_version >= '3.12'" }, + { name = "attrs" }, + { name = "pathspec" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/e4/40998faefc72ba1ddeb640a44fba92935353525dba110488806da8339c0b/scantree-0.0.4.tar.gz", hash = "sha256:15bd5cb24483b04db2c70653604e8ea3522e98087db7e38ab8482f053984c0ac", size = 24643, upload-time = "2024-08-03T20:08:59.413Z" } wheels = [ @@ -4287,8 +4289,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "jeepney", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "cryptography" }, + { name = "jeepney" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -4442,10 +4444,10 @@ name = "storage3" version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "deprecation", marker = "python_full_version >= '3.12'" }, - { name = "httpx", extra = ["http2"], marker = "python_full_version >= '3.12'" }, - { name = "pydantic", marker = "python_full_version >= '3.12'" }, - { name = "yarl", marker = "python_full_version >= '3.12'" }, + { name = "deprecation" }, + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "yarl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/13/30/fee43d523d3f680a833a4aae5bf8094de0b9031b0c2bddb3e0bc6e829e1b/storage3-2.31.0.tar.gz", hash = "sha256:d2161e2ea650dc115a1787c30e09b118365589ac772f4dd8643e3a503ecfc667", size = 20348, upload-time = "2026-06-04T13:37:23.703Z" } wheels = [ @@ -4466,13 +4468,13 @@ name = "supabase" version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx", marker = "python_full_version >= '3.12'" }, - { name = "postgrest", marker = "python_full_version >= '3.12'" }, - { name = "realtime", marker = "python_full_version >= '3.12'" }, - { name = "storage3", marker = "python_full_version >= '3.12'" }, - { name = "supabase-auth", marker = "python_full_version >= '3.12'" }, - { name = "supabase-functions", marker = "python_full_version >= '3.12'" }, - { name = "yarl", marker = "python_full_version >= '3.12'" }, + { name = "httpx" }, + { name = "postgrest" }, + { name = "realtime" }, + { name = "storage3" }, + { name = "supabase-auth" }, + { name = "supabase-functions" }, + { name = "yarl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fe/8e/54a2f950629689b1613434a61fc3bff5f92f84ba6b20213f5b2add05c1bb/supabase-2.31.0.tar.gz", hash = "sha256:3467b09d00482b9a0138235bdbde7a350426f93cf2a1342372eaddfc669f1206", size = 9805, upload-time = "2026-06-04T13:37:25.22Z" } wheels = [ @@ -4484,9 +4486,9 @@ name = "supabase-auth" version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx", extra = ["http2"], marker = "python_full_version >= '3.12'" }, - { name = "pydantic", marker = "python_full_version >= '3.12'" }, - { name = "pyjwt", extra = ["crypto"], marker = "python_full_version >= '3.12'" }, + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "pyjwt", extra = ["crypto"] }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/8a/408689cf39820f0d46d2731d6747ff94dbefc87ae977b4b5c4066da5b070/supabase_auth-2.31.0.tar.gz", hash = "sha256:0945b33fa96239c76dc8eaf96d7d2c94991950d24b4cfe4a5c2da9aa5e909663", size = 39151, upload-time = "2026-06-04T13:37:27.375Z" } wheels = [ @@ -4498,9 +4500,9 @@ name = "supabase-functions" version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx", extra = ["http2"], marker = "python_full_version >= '3.12'" }, - { name = "strenum", marker = "python_full_version >= '3.12'" }, - { name = "yarl", marker = "python_full_version >= '3.12'" }, + { name = "httpx", extra = ["http2"] }, + { name = "strenum" }, + { name = "yarl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/84/5d/61c2446ed26a57fa5543f9c270a731320911569202340d15341f72cdba7c/supabase_functions-2.31.0.tar.gz", hash = "sha256:4ad027b3ae3bd28b31233339f4db1da6965affd3546f655b421baf40cee2690f", size = 4683, upload-time = "2026-06-04T13:37:28.878Z" } wheels = [ @@ -4731,26 +4733,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.21" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ee/20/2ba8fd9493c89c41dfe9dbb73bc70a28b28028463bc0d2897ba8be36230a/ty-0.0.21.tar.gz", hash = "sha256:a4c2ba5d67d64df8fcdefd8b280ac1149d24a73dbda82fa953a0dff9d21400ed", size = 5297967, upload-time = "2026-03-06T01:57:13.809Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/36/70/edf38bb37517531681d1c37f5df64744e5ad02673c02eb48447eae4bea08/ty-0.0.21-py3-none-linux_armv6l.whl", hash = "sha256:7bdf2f572378de78e1f388d24691c89db51b7caf07cf90f2bfcc1d6b18b70a76", size = 10299222, upload-time = "2026-03-06T01:57:16.64Z" }, - { url = "https://files.pythonhosted.org/packages/72/62/0047b0bd19afeefbc7286f20a5f78a2aa39f92b4d89853f0d7185ab89edc/ty-0.0.21-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7e9613994610431ab8625025bd2880dbcb77c5c9fabdd21134cda12d840a529d", size = 10130513, upload-time = "2026-03-06T01:57:29.93Z" }, - { url = "https://files.pythonhosted.org/packages/a2/20/0b93a9e91aaed23155780258cdfdb4726ef68b6985378ac069bc427291a0/ty-0.0.21-py3-none-macosx_11_0_arm64.whl", hash = "sha256:56d3b198b64dd0a19b2b66e257deaed2ecea568e722ae5352f3c6fb62027f89d", size = 9605425, upload-time = "2026-03-06T01:57:27.115Z" }, - { url = "https://files.pythonhosted.org/packages/ea/fd/9945e2fa2996a1287b1e1d7ce050e97e1f420233b271e770934bfa0880a0/ty-0.0.21-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d23d2c34f7a77d974bb08f0860ef700addc8a683d81a0319f71c08f87506cfd0", size = 10108298, upload-time = "2026-03-06T01:57:35.429Z" }, - { url = "https://files.pythonhosted.org/packages/52/e7/4ec52fcb15f3200826c9f048472c062549a05b0d1ef0b51f32d527b513c4/ty-0.0.21-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56b01fd2519637a4ca88344f61c96225f540c98ff18bca321d4eaa7bb0f7aa2f", size = 10121556, upload-time = "2026-03-06T01:57:03.242Z" }, - { url = "https://files.pythonhosted.org/packages/ee/c0/ad457be2a8abea0f25549598bd098554540ced66229488daa0d558dad3c8/ty-0.0.21-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e9de7e11c63c6afc40f3e9ba716374add171aee7fabc70b5146a510705c6d41b", size = 10603264, upload-time = "2026-03-06T01:56:52.134Z" }, - { url = "https://files.pythonhosted.org/packages/f8/5b/2ecc7a2175243a4bcb72f5298ae41feabbb93b764bb0dc45722f3752c2c2/ty-0.0.21-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:62f7f5b235c4f7876db305c36997aea07b7af29b1a068f373d0e2547e25f32ff", size = 11196428, upload-time = "2026-03-06T01:57:32.94Z" }, - { url = "https://files.pythonhosted.org/packages/37/f5/aff507d6a901f328ef96a298032b0c11aaaf950a146ed7dd3b5bf2cd3acf/ty-0.0.21-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ee8399f7c453a425291e6688efe430cfae7ab0ac4ffd50eba9f872bf878b54f6", size = 10866355, upload-time = "2026-03-06T01:56:57.831Z" }, - { url = "https://files.pythonhosted.org/packages/be/30/822bbcb92d55b65989aa7ed06d9585f28ade9c9447369194ed4b0fb3b5b9/ty-0.0.21-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210e7568c9f886c4d01308d751949ee714ad7ad9d7d928d2ba90d329dd880367", size = 10738177, upload-time = "2026-03-06T01:57:11.256Z" }, - { url = "https://files.pythonhosted.org/packages/57/cc/46e7991b6469e93ac2c7e533a028983e402485580150ac864c56352a3a82/ty-0.0.21-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:53508e345b11569f78b21ba8e2b4e61df38a9754947fb3cd9f2ef574367338fb", size = 10079158, upload-time = "2026-03-06T01:57:00.516Z" }, - { url = "https://files.pythonhosted.org/packages/15/c2/0bbdadfbd008240f8f1a87dc877433cb3884436097926107ccf06e618199/ty-0.0.21-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:553e43571f4a35604c36cfd07d8b61a5eb7a714e3c67f8c4ff2cf674fefbaef9", size = 10150535, upload-time = "2026-03-06T01:57:08.815Z" }, - { url = "https://files.pythonhosted.org/packages/c5/b5/2dbdb7b57b5362200ef0a39738ebd31331726328336def0143ac097ee59d/ty-0.0.21-py3-none-musllinux_1_2_i686.whl", hash = "sha256:666f6822e3b9200abfa7e95eb0ddd576460adb8d66b550c0ad2c70abc84a2048", size = 10319803, upload-time = "2026-03-06T01:57:19.106Z" }, - { url = "https://files.pythonhosted.org/packages/72/84/70e52c0b7abc7c2086f9876ef454a73b161d3125315536d8d7e911c94ca4/ty-0.0.21-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a0854d008347ce4a5fb351af132f660a390ab2a1163444d075251d43e6f74b9b", size = 10826239, upload-time = "2026-03-06T01:57:21.727Z" }, - { url = "https://files.pythonhosted.org/packages/a1/8a/1f72480fd013bbc6cd1929002abbbcde9a0b08ead6a15154de9d7f7fa37e/ty-0.0.21-py3-none-win32.whl", hash = "sha256:bef3ab4c7b966bcc276a8ac6c11b63ba222d21355b48d471ea782c4104eee4e0", size = 9693196, upload-time = "2026-03-06T01:57:24.126Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f8/1104808b875c26c640e536945753a78562d606bef4e241d9dbf3d92477f6/ty-0.0.21-py3-none-win_amd64.whl", hash = "sha256:a709d576e5bea84b745d43058d8b9cd4f27f74a0b24acb4b0cbb7d3d41e0d050", size = 10668660, upload-time = "2026-03-06T01:56:55.06Z" }, - { url = "https://files.pythonhosted.org/packages/1b/b8/25e0adc404bbf986977657b25318991f93097b49f8aea640d93c0b0db68e/ty-0.0.21-py3-none-win_arm64.whl", hash = "sha256:f72047996598ac20553fb7e21ba5741e3c82dee4e9eadf10d954551a5fe09391", size = 10104161, upload-time = "2026-03-06T01:57:06.072Z" }, +version = "0.0.63" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/ce/cbeaa5c7576fec643609dfbf200d59493523b1cc0481d4e7a5effcbf0630/ty-0.0.63.tar.gz", hash = "sha256:c2f66439393b3acac69306c117d4ae44638ce5fffa4a20c21046e85bd473359f", size = 6280695, upload-time = "2026-07-23T11:41:39.845Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/e4/d17a8e113ab15c692fe6fb9c422112d4bee7e829d80ced35826b45d98d98/ty-0.0.63-py3-none-linux_armv6l.whl", hash = "sha256:9a4ef7782e3af314fb63d006bec5ac3025bd70fee774b4dcfb5e44e8564f1994", size = 12056302, upload-time = "2026-07-23T11:41:03.5Z" }, + { url = "https://files.pythonhosted.org/packages/be/0b/357234c815dc4bfcc88c3f860aa0983fe4228c8aa2a5bb16c35ee08a94af/ty-0.0.63-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:01eab0ab70d51ad10298aa2d4b058b387a1fe93e5ee52d2a1ee23e9c69ba8354", size = 11737674, upload-time = "2026-07-23T11:41:05.862Z" }, + { url = "https://files.pythonhosted.org/packages/1c/13/193d9aeeb6774690351cff9fafabd3ae9b54cc225d125e07fa004ce23bdc/ty-0.0.63-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a671b61eaad16178389e05b9c108c9cb75ae8d84968fe55906484f55d6268338", size = 11264191, upload-time = "2026-07-23T11:41:07.963Z" }, + { url = "https://files.pythonhosted.org/packages/4f/b7/c5843e16759a2b25fc8a7197473474038e585ac9fdaa6fa16aeb6384bf6a/ty-0.0.63-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b255cc83d95c51bb9ee4931303fdbece8cb1d6d7c655eb77a3f2c8f349fb64d6", size = 11818890, upload-time = "2026-07-23T11:41:09.885Z" }, + { url = "https://files.pythonhosted.org/packages/06/20/adf83ae1fc570d9bb449605e1eeeaa523ad2329ca838a636698b5c135d11/ty-0.0.63-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0da1e8183aa4f893421a173904478021498f542ee41273660538da6649a9631c", size = 11853141, upload-time = "2026-07-23T11:41:12.151Z" }, + { url = "https://files.pythonhosted.org/packages/ea/90/f8effd846e3ee13486ea08257c13094d58b9f188c5641bf609d6a7d5c09f/ty-0.0.63-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:128f38eb5a67199e3811426386f7ec96f41251ddf45e04ddc0bcbb29d4853245", size = 12545184, upload-time = "2026-07-23T11:41:14.4Z" }, + { url = "https://files.pythonhosted.org/packages/b8/76/aac3a30d40431eb1d329acea016b248f5b6255e30ef3d28e19447e344be2/ty-0.0.63-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bedf34aff8b0557f2a7119b314a68a9c9e58c1d21554d0e2748f2c5fe6a1f638", size = 13062375, upload-time = "2026-07-23T11:41:16.441Z" }, + { url = "https://files.pythonhosted.org/packages/62/3d/0158733932893e17f6008dadd74c8d7aed422fefc66e47fe77888014fe8c/ty-0.0.63-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7328d63c34587606dce02935a25404f54cd0161bfe413b8f7fc21d174aead612", size = 12619461, upload-time = "2026-07-23T11:41:18.538Z" }, + { url = "https://files.pythonhosted.org/packages/f5/be/e280ad095b050778f16f493d49a073fa5f6d8f301d3e2e59be6a672ba05c/ty-0.0.63-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504c4457f3a62afe836c1f26a2c9a12549299095f9cc4146778558df51a7515c", size = 12376402, upload-time = "2026-07-23T11:41:20.482Z" }, + { url = "https://files.pythonhosted.org/packages/57/57/619788bf335cb86b090470b557ae1eed33613dc24e12142505d32b462e58/ty-0.0.63-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:7394ed39424b027c89d5f0c818871c791e33958b88f5bb13e14bd4a12a3ca631", size = 12671377, upload-time = "2026-07-23T11:41:22.489Z" }, + { url = "https://files.pythonhosted.org/packages/07/a2/0aff2cdd3c98e2c4729337542b8da0ce1980790e1600e0c4a717a1f46293/ty-0.0.63-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:94c3f155230490f8fb505911655e940c630c25cc0c35a76690cb413254fb4437", size = 11768090, upload-time = "2026-07-23T11:41:24.558Z" }, + { url = "https://files.pythonhosted.org/packages/43/4a/cdb5f1d26154144dfee08e1bd671daf827238170f7cee9dad43990bb2016/ty-0.0.63-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:c16e1d8b4f0ae99106d5f13a894034a202baf6cdab61e2bb1a239de82904f839", size = 11867747, upload-time = "2026-07-23T11:41:26.794Z" }, + { url = "https://files.pythonhosted.org/packages/7e/26/ecc09ecb70bc9fbfdf42fa57ff29568f173e8200df575a0d72dc2b8486f9/ty-0.0.63-py3-none-musllinux_1_2_i686.whl", hash = "sha256:b66293dad89eaed4b9fbf3b661614dbeb85b59ba037178a3f9a0adedf264da5c", size = 12120000, upload-time = "2026-07-23T11:41:28.731Z" }, + { url = "https://files.pythonhosted.org/packages/14/07/862822f9c2c397785b69b25cf79b4dfc3c0d55684b9adf11ac194450f8e1/ty-0.0.63-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:8b29cc832e2ec73502c97dd7325d6ae0e085b34a33529a0f785192317d9862fc", size = 12477862, upload-time = "2026-07-23T11:41:30.847Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c2/50b08b641578d6f48e8aa28a91d0de32c91aa24dd926ed199e8afd2d37ea/ty-0.0.63-py3-none-win32.whl", hash = "sha256:f0a8fbfd1f990c0c5d85cce018d438969ac79e49247e2192c9745394bb7d17ab", size = 11433155, upload-time = "2026-07-23T11:41:33.28Z" }, + { url = "https://files.pythonhosted.org/packages/92/2d/d422a5568f0d1f317186be22241288dc6e08b71dc24aca223f22038ac2d2/ty-0.0.63-py3-none-win_amd64.whl", hash = "sha256:2aa2370bdd6f42e9f37518c812379bef70b9162f9e5aad511495229b0b71cb93", size = 12450036, upload-time = "2026-07-23T11:41:35.559Z" }, + { url = "https://files.pythonhosted.org/packages/35/97/2c9748e28ead0650c7ad3e5f74f178832ceabd7cb5c272a882f29eb32ee4/ty-0.0.63-py3-none-win_arm64.whl", hash = "sha256:95ac1a62162c3c7ac204731e95ebf766d62a46a7bfa238a6acbe923fcb772cb1", size = 11826243, upload-time = "2026-07-23T11:41:37.749Z" }, ] [[package]] @@ -5025,10 +5028,10 @@ dev = [ { name = "pytest-xdist", specifier = ">=3.8.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, { name = "reasoning-gym", specifier = ">=0.1.8" }, - { name = "ruff", specifier = ">=0.9.0" }, + { name = "ruff", specifier = "==0.16.0" }, { name = "stagehand", specifier = ">=3.4.0" }, { name = "textarena", specifier = "==0.7.4" }, - { name = "ty", specifier = ">=0.0.1a29,<0.0.22" }, + { name = "ty", specifier = "==0.0.63" }, ] examples = [ { name = "alphabet-sort-v1", editable = "environments/alphabet_sort_v1" }, diff --git a/verifiers/clients/nemorl_chat_completions_client.py b/verifiers/clients/nemorl_chat_completions_client.py index 99f02b79b2..90ba123265 100644 --- a/verifiers/clients/nemorl_chat_completions_client.py +++ b/verifiers/clients/nemorl_chat_completions_client.py @@ -108,8 +108,8 @@ def _attach_trajectory_tokens_to_prompt( continue msg = prompt[i] if (prompt_ids := tokens.get("prompt_ids")) is not None: - msg.prompt_token_ids = list(prompt_ids) + msg.prompt_token_ids = list(prompt_ids) # ty:ignore[invalid-assignment] if (completion_ids := tokens.get("completion_ids")) is not None: - msg.generation_token_ids = list(completion_ids) + msg.generation_token_ids = list(completion_ids) # ty:ignore[invalid-assignment] if (completion_logprobs := tokens.get("completion_logprobs")) is not None: - msg.generation_log_probs = list(completion_logprobs) + msg.generation_log_probs = list(completion_logprobs) # ty:ignore[invalid-assignment] diff --git a/verifiers/clients/openai_chat_completions_client.py b/verifiers/clients/openai_chat_completions_client.py index 8f33277764..2745b29127 100644 --- a/verifiers/clients/openai_chat_completions_client.py +++ b/verifiers/clients/openai_chat_completions_client.py @@ -233,7 +233,7 @@ def from_chat_message(message: Message) -> OpenAIChatMessage: role="assistant", content=cast(Any, normalize_content(message.content)), tool_calls=cast(Any, oai_tool_calls), - reasoning_content=message.reasoning_content, # type: ignore[arg-type] + reasoning_content=message.reasoning_content, # type: ignore[arg-type] # ty:ignore[invalid-key] ) elif isinstance(message, ToolMessage): return ChatCompletionToolMessageParam( diff --git a/verifiers/clients/openai_chat_completions_token_client.py b/verifiers/clients/openai_chat_completions_token_client.py index 5068986f23..04fcd6fcaa 100644 --- a/verifiers/clients/openai_chat_completions_token_client.py +++ b/verifiers/clients/openai_chat_completions_token_client.py @@ -290,7 +290,7 @@ async def find_largest_prefix_match() -> tuple[list[int], bool, int] | None: if tool_call_ids: dummy_assistant: OpenAIChatMessage = ChatCompletionAssistantMessageParam( role="assistant", - reasoning_content="", # type: ignore[typeddict-unknown-key] + reasoning_content="", # type: ignore[typeddict-unknown-key] # ty:ignore[invalid-key] tool_calls=[ ChatCompletionMessageFunctionToolCallParam( id=tc_id, @@ -303,7 +303,7 @@ async def find_largest_prefix_match() -> tuple[list[int], bool, int] | None: else: dummy_assistant: OpenAIChatMessage = ChatCompletionAssistantMessageParam( role="assistant", - reasoning_content="", # type: ignore[typeddict-unknown-key] + reasoning_content="", # type: ignore[typeddict-unknown-key] # ty:ignore[invalid-key] content="x", ) diff --git a/verifiers/envs/env_group.py b/verifiers/envs/env_group.py index 5be67ea22c..f0d2d4c08a 100644 --- a/verifiers/envs/env_group.py +++ b/verifiers/envs/env_group.py @@ -294,7 +294,7 @@ def add_example_id(example, i): return dataset @final - async def run_rollout( # type: ignore[override] + async def run_rollout( # type: ignore[override] # ty:ignore[override-of-final-method] self, input: RolloutInput, client: Client | ClientConfig, @@ -330,10 +330,10 @@ async def run_rollout( # type: ignore[override] state_columns, env.env_client, ) - return _set_info_route(output, route) # type: ignore[return-value] + return _set_info_route(output, route) # type: ignore[return-value] # ty:ignore[invalid-return-type] @final - async def run_group( # type: ignore[override] + async def run_group( # type: ignore[override] # ty:ignore[override-of-final-method] self, group_inputs: list[RolloutInput], client: Client | ClientConfig, @@ -342,7 +342,7 @@ async def run_group( # type: ignore[override] max_retries: int = 0, state_columns: list[str] | None = None, env_client: EnvClient | None = None, - ) -> list[vf.RolloutOutput]: + ) -> list[vf.RolloutOutput]: # ty:ignore[invalid-method-override] target_env_client = env_client or self.env_client if target_env_client is not None: if not isinstance(client, ClientConfig): @@ -383,7 +383,7 @@ async def run_group( # type: ignore[override] state_columns, env.env_client, ) - return [_set_info_route(output, route) for output in outputs] # type: ignore[return-value] + return [_set_info_route(output, route) for output in outputs] # type: ignore[return-value] # ty:ignore[invalid-return-type] @final async def rollout( @@ -417,7 +417,7 @@ def _route_child_input( child_input["info"] = info else: child_input.pop("info", None) - return env_name, child_input, route # type: ignore[return-value] + return env_name, child_input, route # type: ignore[return-value] # ty:ignore[invalid-return-type] def _input_env_route(self, input: RolloutInput) -> tuple[str, ...]: info = _info_dict(input.get("info")) diff --git a/verifiers/envs/environment.py b/verifiers/envs/environment.py index 80e0dd4987..b4a432b074 100644 --- a/verifiers/envs/environment.py +++ b/verifiers/envs/environment.py @@ -162,7 +162,7 @@ def __init__( if dataset is not None: if callable(dataset): - self.dataset_source: DatasetBuilder | None = cast( + self.dataset_source: DatasetBuilder | None = cast( # ty:ignore[unsupported-operator] DatasetBuilder, dataset ) else: @@ -173,7 +173,7 @@ def __init__( if eval_dataset is not None: if callable(eval_dataset): - self.eval_dataset_source: DatasetBuilder | None = cast( + self.eval_dataset_source: DatasetBuilder | None = cast( # ty:ignore[unsupported-operator] DatasetBuilder, eval_dataset ) else: @@ -294,7 +294,7 @@ def format_prompt_fn(prompt_str: str) -> Messages: if few_shot: messages.extend(few_shot) messages.append({"role": "user", "content": prompt_str}) - return messages + return messages # ty:ignore[invalid-return-type] if answer_key == "answer": dataset = dataset.map( diff --git a/verifiers/envs/experimental/composable/task.py b/verifiers/envs/experimental/composable/task.py index 36251a80e3..6e0d67b366 100644 --- a/verifiers/envs/experimental/composable/task.py +++ b/verifiers/envs/experimental/composable/task.py @@ -369,7 +369,7 @@ async def validate( try: from tqdm.auto import tqdm except ImportError: # pragma: no cover - tqdm = None # type: ignore[assignment] + tqdm = None # type: ignore[assignment] # ty:ignore[invalid-assignment] import verifiers as vf @@ -521,7 +521,7 @@ async def _validate_once(i: int) -> tuple[bool, BaseException | None, dict]: state: State = { # type: ignore[assignment] "info": info, "answer": row.get("answer", ""), - } + } # ty:ignore[invalid-assignment] try: valid = await self.validate_instance(state) return valid, None, state @@ -543,7 +543,7 @@ async def _validate_once(i: int) -> tuple[bool, BaseException | None, dict]: state: State = { # type: ignore[assignment] "info": info, "answer": row.get("answer", ""), - } + } # ty:ignore[invalid-assignment] spec = self.get_sandbox_spec(info) # validate() runs without a SandboxMixin, so resolve # spec.timeout_minutes=None (its "auto-derive at rollout diff --git a/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py b/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py index ca39b3e536..d68d617e55 100644 --- a/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py +++ b/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py @@ -319,7 +319,7 @@ async def run_single_test(i, test_case_inputs, test_case_outputs): ) try: if isinstance(exec_outputs[0], tuple): - exec_outputs = [list(x) for x in exec_outputs] + exec_outputs = [list(x) for x in exec_outputs] # ty:ignore[invalid-argument-type] tmp_result = tmp_result or ( exec_outputs == test_case_outputs_parsed[0] ) diff --git a/verifiers/envs/experimental/harbor_env/mcp.py b/verifiers/envs/experimental/harbor_env/mcp.py index 4c2d006019..958bc977bb 100644 --- a/verifiers/envs/experimental/harbor_env/mcp.py +++ b/verifiers/envs/experimental/harbor_env/mcp.py @@ -180,7 +180,7 @@ async def stop_mcp_servers(self, state: vf.State) -> None: for name in list(jobs): try: pid_file = shlex.quote(self._mcp_pid_file(name)) - await self.sandbox_client.execute_command( # type: ignore[attr-defined] + await self.sandbox_client.execute_command( # type: ignore[attr-defined] # ty:ignore[unresolved-attribute] sandbox_id, f'kill -9 -"$(cat {pid_file} 2>/dev/null)" 2>/dev/null; rm -f {pid_file}', working_dir=None, @@ -203,7 +203,7 @@ async def _start_mcp_server( f"MCP server {server.name!r} has no port in its URL {server.url!r}" ) - job = await self.sandbox_client.start_background_job( # type: ignore[attr-defined] + job = await self.sandbox_client.start_background_job( # type: ignore[attr-defined] # ty:ignore[unresolved-attribute] sandbox_id=sandbox_id, command=( f"setsid sh -c {shlex.quote(command)} & " @@ -239,7 +239,7 @@ async def _wait_for_mcp_server( consecutive_failures = 0 while True: - status = await self.sandbox_client.get_background_job( # type: ignore[attr-defined] + status = await self.sandbox_client.get_background_job( # type: ignore[attr-defined] # ty:ignore[unresolved-attribute] sandbox_id, job ) if getattr(status, "completed", False): @@ -251,7 +251,7 @@ async def _wait_for_mcp_server( f"Stderr:\n{stderr}" ) - result = await self.sandbox_client.execute_command( # type: ignore[attr-defined] + result = await self.sandbox_client.execute_command( # type: ignore[attr-defined] # ty:ignore[unresolved-attribute] sandbox_id, health_cmd, working_dir=None, timeout=probe_timeout ) if getattr(result, "exit_code", 1) == 0: @@ -280,7 +280,7 @@ async def _wait_for_mcp_server( if consecutive_failures >= hc.retries: log_tail = "" try: - status = await self.sandbox_client.get_background_job( # type: ignore[attr-defined] + status = await self.sandbox_client.get_background_job( # type: ignore[attr-defined] # ty:ignore[unresolved-attribute] sandbox_id, job ) log_tail = (getattr(status, "stderr", "") or "").strip()[-2000:] @@ -313,7 +313,7 @@ async def _patch_mcp_etc_hosts( f"echo {shlex.quote(f'127.0.0.1 {h}')} >> /etc/hosts)" for h in sorted(hosts) ) - result = await self.sandbox_client.execute_command( # type: ignore[attr-defined] + result = await self.sandbox_client.execute_command( # type: ignore[attr-defined] # ty:ignore[unresolved-attribute] sandbox_id, statements, working_dir=None ) exit_code = getattr(result, "exit_code", 0) diff --git a/verifiers/envs/integrations/browser_env/modes/cua_mode.py b/verifiers/envs/integrations/browser_env/modes/cua_mode.py index f4a43ba7d4..8e1ca3b541 100644 --- a/verifiers/envs/integrations/browser_env/modes/cua_mode.py +++ b/verifiers/envs/integrations/browser_env/modes/cua_mode.py @@ -33,10 +33,10 @@ SANDBOX_AVAILABLE = True except ImportError: SANDBOX_AVAILABLE = False - AsyncSandboxClient = None # type: ignore[misc, assignment] - CreateSandboxRequest = None # type: ignore[misc, assignment] - SandboxClient = None # type: ignore[misc, assignment] - APIClient = None # type: ignore[misc, assignment] + AsyncSandboxClient = None # type: ignore[misc, assignment] # ty:ignore[invalid-assignment] + CreateSandboxRequest = None # type: ignore[misc, assignment] # ty:ignore[invalid-assignment] + SandboxClient = None # type: ignore[misc, assignment] # ty:ignore[invalid-assignment] + APIClient = None # type: ignore[misc, assignment] # ty:ignore[invalid-assignment] class CUAMode: @@ -366,7 +366,7 @@ async def _get_sandbox_client(self) -> AsyncSandboxClient: async def _create_sandbox(self) -> str: """Create a new sandbox and return its ID.""" client = await self._get_sandbox_client() - sandbox = await client.create(self._sandbox_request.model_copy()) # type: ignore[union-attr] + sandbox = await client.create(self._sandbox_request.model_copy()) # type: ignore[union-attr] # ty:ignore[unresolved-attribute] self.active_sandboxes.add(sandbox.id) if self.logger: self.logger.debug(f"Created sandbox {sandbox.id}") @@ -375,7 +375,7 @@ async def _create_sandbox(self) -> str: async def _create_sandbox_with_retry(self) -> str: """Create a sandbox with retry, cleaning up orphaned sandboxes from failed attempts.""" previous_sandbox_id: str | None = None - async for attempt in self.retrying: # type: ignore[union-attr] + async for attempt in self.retrying: # type: ignore[union-attr] # ty:ignore[not-iterable] with attempt: if previous_sandbox_id is not None: try: @@ -822,7 +822,7 @@ async def setup_state(self, state: vf.State, **kwargs: Any) -> vf.State: if self._execution_mode == "local": # Local mode: create session via HTTP try: - async for attempt in self.retrying: # type: ignore[union-attr] + async for attempt in self.retrying: # type: ignore[union-attr] # ty:ignore[not-iterable] with attempt: result = await self._create_session_http() session_id = result.get("sessionId") @@ -863,7 +863,7 @@ async def setup_state(self, state: vf.State, **kwargs: Any) -> vf.State: await self._start_server(sandbox_id) await self._wait_for_server(sandbox_id) - async for attempt in self.retrying: # type: ignore[union-attr] + async for attempt in self.retrying: # type: ignore[union-attr] # ty:ignore[not-iterable] with attempt: result = await self._create_session_curl(sandbox_id) session_id = result.get("sessionId") @@ -910,7 +910,7 @@ async def cleanup_session(self, state: vf.State) -> None: # Local mode: destroy session via HTTP if session_id: try: - async for attempt in self.retrying: # type: ignore[union-attr] + async for attempt in self.retrying: # type: ignore[union-attr] # ty:ignore[not-iterable] with attempt: await self._destroy_session_http(session_id) with self._sessions_lock: @@ -926,7 +926,7 @@ async def cleanup_session(self, state: vf.State) -> None: if session_id and sandbox_id: try: - async for attempt in self.retrying: # type: ignore[union-attr] + async for attempt in self.retrying: # type: ignore[union-attr] # ty:ignore[not-iterable] with attempt: await self._destroy_session_curl(session_id, sandbox_id) with self._sessions_lock: @@ -939,7 +939,7 @@ async def cleanup_session(self, state: vf.State) -> None: if sandbox_id: try: - async for attempt in self.retrying: # type: ignore[union-attr] + async for attempt in self.retrying: # type: ignore[union-attr] # ty:ignore[not-iterable] with attempt: await self._delete_sandbox(sandbox_id) except Exception as e: @@ -968,7 +968,7 @@ async def teardown(self, max_concurrent: int = 50) -> None: async def _delete_with_semaphore(session_id: str): async with semaphore: try: - async for attempt in self.retrying: # type: ignore[union-attr] + async for attempt in self.retrying: # type: ignore[union-attr] # ty:ignore[not-iterable] with attempt: await self._destroy_session_http(session_id) with self._sessions_lock: diff --git a/verifiers/envs/integrations/openenv_env.py b/verifiers/envs/integrations/openenv_env.py index d1172b6b86..c9c9834a34 100644 --- a/verifiers/envs/integrations/openenv_env.py +++ b/verifiers/envs/integrations/openenv_env.py @@ -807,7 +807,7 @@ def _single_string_field(self, schema: dict[str, Any]) -> str | None: if len(props) == 1: field_name, spec = next(iter(props.items())) if isinstance(spec, dict) and spec.get("type") == "string": - return field_name + return field_name # ty:ignore[invalid-return-type] return None def _render_observation_messages( diff --git a/verifiers/envs/integrations/textarena_env.py b/verifiers/envs/integrations/textarena_env.py index 7a438b7437..61dbf62452 100644 --- a/verifiers/envs/integrations/textarena_env.py +++ b/verifiers/envs/integrations/textarena_env.py @@ -104,7 +104,7 @@ def build_shared_memo(ta_env) -> dict: async def setup_state(self, state: vf.State, **kwargs) -> vf.State: ta_env = await asyncio.to_thread(deepcopy, self.ta_env, self.shared_memo.copy()) - ta_env.state.game_state["secret_word"] = state["answer"] # type: ignore[unresolved-attribute] + ta_env.state.game_state["secret_word"] = state["answer"] # type: ignore[unresolved-attribute] # ty:ignore[unresolved-attribute] state["ta_env"] = ta_env return state diff --git a/verifiers/envs/tool_env.py b/verifiers/envs/tool_env.py index ed057b5fa0..ca02f7d0c3 100644 --- a/verifiers/envs/tool_env.py +++ b/verifiers/envs/tool_env.py @@ -33,7 +33,7 @@ def remove_tool_metric(self, tool_name: str): self.tool_names.remove(tool_name) metric_name = f"{tool_name}_calls" for i, func in enumerate(self.funcs): - if func.__name__ == metric_name: + if func.__name__ == metric_name: # ty:ignore[unresolved-attribute] self.funcs.pop(i) self.weights.pop(i) break diff --git a/verifiers/gepa/adapter.py b/verifiers/gepa/adapter.py index cd8e72412d..485ae3f6e7 100644 --- a/verifiers/gepa/adapter.py +++ b/verifiers/gepa/adapter.py @@ -184,7 +184,7 @@ def _inject_system_prompt( if isinstance(prompt, str): inp_copy["prompt"] = f"{system_prompt}\n\n{prompt}" else: - prompt = [dict(m) for m in prompt] + prompt = [dict(m) for m in prompt] # ty:ignore[not-iterable] if not prompt: # Empty prompt list - just add system message prompt = [{"role": "system", "content": system_prompt}] @@ -195,7 +195,7 @@ def _inject_system_prompt( inp_copy["prompt"] = prompt modified.append(inp_copy) - return modified + return modified # ty:ignore[invalid-return-type] def _extract_user_query(prompt: Messages) -> str: diff --git a/verifiers/rubrics/experimental/hybrid_math_rubric.py b/verifiers/rubrics/experimental/hybrid_math_rubric.py index ddec7212a7..00ec523114 100644 --- a/verifiers/rubrics/experimental/hybrid_math_rubric.py +++ b/verifiers/rubrics/experimental/hybrid_math_rubric.py @@ -129,7 +129,7 @@ def __init__( self.add_reward_func(self.correct_answer, weight=1) self.judge_model = judge_model if use_judge_fallback else None - self.class_objects["judge_model"] = self.judge_model + self.class_objects["judge_model"] = self.judge_model # ty:ignore[invalid-assignment] # Delegate math verification to default MathRubric # We clear its auto-registered reward func since we manage scoring ourselves diff --git a/verifiers/rubrics/rubric.py b/verifiers/rubrics/rubric.py index 6c45d7a4f3..603c20984d 100644 --- a/verifiers/rubrics/rubric.py +++ b/verifiers/rubrics/rubric.py @@ -168,7 +168,7 @@ def group_score_objects(self, states: list[State]) -> dict[str, Any]: **self.class_objects, ) for provider in self.group_score_object_providers: - objects.update(provider(states)) + objects.update(provider(states)) # ty:ignore[no-matching-overload] return objects def task_for_state(self, state: State, resources: object | None) -> object: @@ -203,7 +203,7 @@ def func(completion, answer, **kwargs): ans = float(await maybe_await(func, **merged)) except Exception as e: self.logger.error( - f"Error calling reward function {func.__name__}: {e}" # type: ignore[unresolved-attribute] + f"Error calling reward function {func.__name__}: {e}" # type: ignore[unresolved-attribute] # ty:ignore[unresolved-attribute] ) ans = 0.0 else: @@ -212,7 +212,7 @@ def func(completion, answer, **kwargs): ans = float(await maybe_await(func, **allowed)) except Exception as e: self.logger.error( - f"Error calling reward function {func.__name__}: {e}" # type: ignore[unresolved-attribute] + f"Error calling reward function {func.__name__}: {e}" # type: ignore[unresolved-attribute] # ty:ignore[unresolved-attribute] ) ans = 0.0 return ans @@ -248,7 +248,7 @@ async def _call_group_reward_func( ans = await maybe_await(func, **merged) except Exception as e: self.logger.error( - f"Error calling group reward function {func.__name__}: {e}" # type: ignore[unresolved-attribute] + f"Error calling group reward function {func.__name__}: {e}" # type: ignore[unresolved-attribute] # ty:ignore[unresolved-attribute] ) ans = [0.0] * len(states) else: @@ -257,7 +257,7 @@ async def _call_group_reward_func( ans = await maybe_await(func, **allowed) except Exception as e: self.logger.error( - f"Error calling group reward function {func.__name__}: {e}" # type: ignore[unresolved-attribute] + f"Error calling group reward function {func.__name__}: {e}" # type: ignore[unresolved-attribute] # ty:ignore[unresolved-attribute] ) ans = [0.0] * len(states) return ans @@ -335,7 +335,7 @@ async def score_rollout(self, state: State): ) rewards = RolloutScore( metrics={ - func.__name__: reward + func.__name__: reward # ty:ignore[unresolved-attribute] for func, reward in zip(reward_funcs, reward_scores) }, reward=sum( @@ -380,7 +380,7 @@ async def score_group(self, states: list[State]): # GroupRewardFunc: score all states together group_func = cast(GroupRewardFunc, func) scores = await self._call_group_reward_func(group_func, states) - func_name = func.__name__ + func_name = func.__name__ # ty:ignore[unresolved-attribute] if func_name not in aggregated_metrics: aggregated_metrics[func_name] = [0.0] * num_states for i in range(num_states): @@ -395,7 +395,7 @@ async def score_group(self, states: list[State]): ] scores = await asyncio.gather(*score_tasks) - func_name = func.__name__ + func_name = func.__name__ # ty:ignore[unresolved-attribute] if func_name not in aggregated_metrics: aggregated_metrics[func_name] = [0.0] * num_states for i in range(num_states): diff --git a/verifiers/scripts/gepa.py b/verifiers/scripts/gepa.py index 053c35e21a..682299e5a2 100644 --- a/verifiers/scripts/gepa.py +++ b/verifiers/scripts/gepa.py @@ -829,7 +829,7 @@ def run_gepa_optimization( logger.debug(f"Results saved to {run_dir}") # Set result info for final summary - best_prompt = result.best_candidate.get("system_prompt", "") # type: ignore[unresolved-attribute] + best_prompt = result.best_candidate.get("system_prompt", "") # type: ignore[unresolved-attribute] # ty:ignore[unresolved-attribute] display.set_result(best_prompt=best_prompt, save_path=save_path) return result diff --git a/verifiers/types.py b/verifiers/types.py index 17f6c1e04c..87aa10a7b2 100644 --- a/verifiers/types.py +++ b/verifiers/types.py @@ -479,7 +479,7 @@ def __getitem__(self, key: str) -> Any: return input_obj[key] return super().__getitem__(key) - def get(self, key: str, default: Any = None) -> Any: + def get(self, key: str, default: Any = None) -> Any: # ty:ignore[invalid-method-override] try: return self[key] except KeyError: @@ -506,7 +506,7 @@ def pop(self, key: str) -> Any: ... @overload def pop(self, key: str, default: _DefaultValue) -> Any | _DefaultValue: ... - def pop(self, key: str, default: Any = _MISSING) -> Any: + def pop(self, key: str, default: Any = _MISSING) -> Any: # ty:ignore[invalid-method-override] if default is _MISSING: return super().pop(key) return super().pop(key, default) diff --git a/verifiers/utils/async_utils.py b/verifiers/utils/async_utils.py index a22df5fd69..1b9513c52e 100644 --- a/verifiers/utils/async_utils.py +++ b/verifiers/utils/async_utils.py @@ -184,7 +184,7 @@ def reraise_error_from_state(result, error_types: tuple[type[Exception], ...]): def log_retry(retry_state: tc.RetryCallState) -> None: """Log a warning with the exception and the number of attempts.""" - caller = retry_state.fn.__name__ if retry_state.fn else "unknown function" + caller = retry_state.fn.__name__ if retry_state.fn else "unknown function" # ty:ignore[unresolved-attribute] error_chain = ( repr( ErrorChain( @@ -203,7 +203,7 @@ def log_retry(retry_state: tc.RetryCallState) -> None: def return_last_result(retry_state: tc.RetryCallState): """Return the last result when retries are exhausted (instead of raising).""" - caller = retry_state.fn.__name__ if retry_state.fn else "unknown function" + caller = retry_state.fn.__name__ if retry_state.fn else "unknown function" # ty:ignore[unresolved-attribute] error_chain = ( repr( ErrorChain( diff --git a/verifiers/utils/eval_utils.py b/verifiers/utils/eval_utils.py index e1cdabf72c..159e6bfde8 100644 --- a/verifiers/utils/eval_utils.py +++ b/verifiers/utils/eval_utils.py @@ -424,10 +424,10 @@ def _expand_ablation(ablation: dict, global_defaults: dict) -> list[dict]: for key, value in zip(keys, combo): if key.startswith("env_args."): env_key = key[len("env_args.") :] - config["env_args"] = {**config.get("env_args", {}), env_key: value} + config["env_args"] = {**config.get("env_args", {}), env_key: value} # ty:ignore[invalid-argument-type] elif key.startswith("args."): env_key = key[len("args.") :] - config["args"] = {**config.get("args", {}), env_key: value} + config["args"] = {**config.get("args", {}), env_key: value} # ty:ignore[invalid-argument-type] else: config[key] = value expanded.append(config) @@ -1168,7 +1168,7 @@ async def run_evaluations(config: EvalRunConfig) -> None: from verifiers.utils.heartbeat import Heartbeat heart = Heartbeat(config.heartbeat_url) - on_progress = [lambda *_a, **_kw: asyncio.create_task(heart.beat())] + on_progress = [lambda *_a, **_kw: asyncio.create_task(heart.beat())] # ty:ignore[invalid-assignment] start_time = time.time() all_results = await asyncio.gather( @@ -1307,7 +1307,7 @@ def on_display_progress( on_progress: list[ProgressCallback] = [on_display_progress] if heart is not None: - on_progress.append(lambda *_a, **_kw: asyncio.create_task(heart.beat())) + on_progress.append(lambda *_a, **_kw: asyncio.create_task(heart.beat())) # ty:ignore[invalid-argument-type] def on_log(message: str) -> None: display.update_env_state(env_idx, log_message=message) diff --git a/verifiers/utils/interception_utils.py b/verifiers/utils/interception_utils.py index a50d435f02..9094538500 100644 --- a/verifiers/utils/interception_utils.py +++ b/verifiers/utils/interception_utils.py @@ -129,7 +129,7 @@ async def start(self) -> None: ) app.router.add_get( "/health", - lambda _: web.json_response({"status": "ok"}), + lambda _: web.json_response({"status": "ok"}), # ty:ignore[invalid-argument-type] ) runner = web.AppRunner(app) diff --git a/verifiers/utils/message_utils.py b/verifiers/utils/message_utils.py index 7a7cbb86d4..65b7599bcc 100644 --- a/verifiers/utils/message_utils.py +++ b/verifiers/utils/message_utils.py @@ -144,31 +144,31 @@ def normalize_messages( @overload -def get_messages( +def get_messages( # ty:ignore[invalid-overload] messages: Sequence[MessageLike], role: Literal["assistant"] ) -> list[AssistantMessage]: ... @overload -def get_messages( +def get_messages( # ty:ignore[invalid-overload] messages: Sequence[MessageLike], role: Literal["system"] ) -> list[SystemMessage]: ... @overload -def get_messages( +def get_messages( # ty:ignore[invalid-overload] messages: Sequence[MessageLike], role: Literal["user"] ) -> list[UserMessage]: ... @overload -def get_messages( +def get_messages( # ty:ignore[invalid-overload] messages: Sequence[MessageLike], role: Literal["tool"] ) -> list[ToolMessage]: ... @overload -def get_messages( +def get_messages( # ty:ignore[invalid-overload] messages: Sequence[MessageLike], role: Literal["text"] ) -> list[TextMessage]: ... diff --git a/verifiers/utils/save_utils.py b/verifiers/utils/save_utils.py index 8f2acd4088..c21b1a249e 100644 --- a/verifiers/utils/save_utils.py +++ b/verifiers/utils/save_utils.py @@ -428,7 +428,7 @@ def _read_mm_hashes(mm: object) -> dict[str, list[str]]: return {} return { modality: list(hs) for modality, hs in hashes.items() if isinstance(hs, list) - } + } # ty:ignore[invalid-return-type] def _is_monotonic_extension( @@ -505,7 +505,7 @@ def _diff_mm_data(mm: object, prior_hashes: dict[str, list[str]]) -> object: keep_idx: list[int] = [] for i, h in enumerate(mod_hashes): if remaining.get(h, 0) > 0: - remaining[h] -= 1 + remaining[h] -= 1 # ty:ignore[invalid-argument-type] else: keep_idx.append(i) if len(keep_idx) != len(mod_hashes): From efad1cd44d5a2f2c9925e3e9436f8191e1498415 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:36:12 +0200 Subject: [PATCH 3/4] Relax Ruff and ty version constraints --- pyproject.toml | 4 ++-- uv.lock | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5aee69a11b..384e3af53d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,9 +57,9 @@ dependencies = [ [dependency-groups] dev = [ - "ruff==0.16.0", + "ruff>=0.16.0", "pre-commit>=2.19.0", - "ty==0.0.63", + "ty>=0.0.63", "pytest>=7.0.0,<9.1.0", "pytest-asyncio>=0.21.0", "pytest-cov>=4.0.0", diff --git a/uv.lock b/uv.lock index f0dc98ecb5..e1672f65d0 100644 --- a/uv.lock +++ b/uv.lock @@ -5028,10 +5028,10 @@ dev = [ { name = "pytest-xdist", specifier = ">=3.8.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, { name = "reasoning-gym", specifier = ">=0.1.8" }, - { name = "ruff", specifier = "==0.16.0" }, + { name = "ruff", specifier = ">=0.16.0" }, { name = "stagehand", specifier = ">=3.4.0" }, { name = "textarena", specifier = "==0.7.4" }, - { name = "ty", specifier = "==0.0.63" }, + { name = "ty", specifier = ">=0.0.63" }, ] examples = [ { name = "alphabet-sort-v1", editable = "environments/alphabet_sort_v1" }, From c8b235475c32f2b1ce9faa8a94737ae8d5c1cef4 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:17:53 +0200 Subject: [PATCH 4/4] Align CI Ruff version with lockfile --- .github/workflows/style.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index ca93b9fd4a..2398c50d2e 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -22,11 +22,11 @@ jobs: - name: Lint uses: astral-sh/ruff-action@v4.1.0 with: - version: "0.15.21" + version: "0.16.0" - name: Format uses: astral-sh/ruff-action@v4.1.0 with: - version: "0.15.21" + version: "0.16.0" args: "format --check" ty: name: Ty