chore: consolidate v1 root modules into utils

tests/v1/conftest.py

@@ -41,8 +41,8 @@
 import verifiers.v1 as vf
 from verifiers.v1.cli.eval.runner import run_eval
 from verifiers.v1.configs.cli.eval import EvalConfig
-from verifiers.v1.loaders import load_environment
 from verifiers.v1.trace import Trace
+from verifiers.v1.utils.loaders import load_environment
 
 # Fixture tasksets/envs (echo-v1, echo-agentic-v1, echo-v0, echo-multi-v0) live in
 # tests/v1/fixtures, added to the path via `pythonpath` in pyproject so the v1 loader and the

tests/v1/test_judges.py

@@ -11,8 +11,8 @@
 import verifiers.v1 as vf
 from verifiers.v1.graph import MessageNode
 from verifiers.v1.judge import Judge, JudgeResponse
-from verifiers.v1.loaders import judge_class, judge_config_type, load_judge
 from verifiers.v1.types import AssistantMessage, UserMessage
+from verifiers.v1.utils.loaders import judge_class, judge_config_type, load_judge
 
 RUBRIC_TOML = """
 [[criteria]]

verifiers/v1/__init__.py

@@ -4,12 +4,6 @@
 
 from verifiers.v1.acp import ACP
 from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent
-from verifiers.v1.artifacts import (
-    ARTIFACTS_DIR,
-    Artifact,
-    collect,
-    restore,
-)
 from verifiers.v1.clients import (
     BaseClientConfig,
     Client,
@@ -34,7 +28,6 @@
 )
 from verifiers.v1.configs.task import TaskConfig
 from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.decorators import metric, reward, stop, tool
 from verifiers.v1.env import Env
 from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig
 from verifiers.v1.episode import Episode, WireEpisode
@@ -59,24 +52,6 @@
     RubricJudge,
     RubricJudgeConfig,
 )
-from verifiers.v1.loaders import (
-    default_harness_id,
-    env_config_type,
-    environment_class,
-    harness_config_type,
-    import_environment,
-    import_harness,
-    import_judge,
-    import_taskset,
-    judge_config_type,
-    load_environment,
-    load_harness,
-    load_judge,
-    load_taskset,
-    resolve_env_config,
-    task_type,
-    taskset_config_type,
-)
 from verifiers.v1.mcp import (
     SharedToolsetConfig,
     Toolset,
@@ -91,24 +66,6 @@
     RuntimeInfo,
     SubprocessConfig,
 )
-from verifiers.v1.scoring import (
-    compare_stdout_results as compare_stdout_results,
-)
-from verifiers.v1.scoring import (
-    extract_boxed_answer as extract_boxed_answer,
-)
-from verifiers.v1.scoring import (
-    parse_judge_choice as parse_judge_choice,
-)
-from verifiers.v1.scoring import (
-    parse_pytest_outcomes as parse_pytest_outcomes,
-)
-from verifiers.v1.scoring import (
-    read_answer_file_or_last_reply as read_answer_file_or_last_reply,
-)
-from verifiers.v1.scoring import (
-    verify_boxed_math_answer as verify_boxed_math_answer,
-)
 from verifiers.v1.state import State, StateT
 from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout, WireTaskData
 from verifiers.v1.taskset import Taskset
@@ -153,6 +110,13 @@
     Usage,
     UserMessage,
 )
+from verifiers.v1.utils.artifacts import (
+    ARTIFACTS_DIR,
+    Artifact,
+    collect,
+    restore,
+)
+from verifiers.v1.utils.decorators import metric, reward, stop, tool
 from verifiers.v1.utils.git import (
     PATCH_CAP_BYTES as PATCH_CAP_BYTES,
 )
@@ -162,6 +126,42 @@
 from verifiers.v1.utils.git import (
     resolve_head as resolve_head,
 )
+from verifiers.v1.utils.loaders import (
+    default_harness_id,
+    env_config_type,
+    environment_class,
+    harness_config_type,
+    import_environment,
+    import_harness,
+    import_judge,
+    import_taskset,
+    judge_config_type,
+    load_environment,
+    load_harness,
+    load_judge,
+    load_taskset,
+    resolve_env_config,
+    task_type,
+    taskset_config_type,
+)
+from verifiers.v1.utils.score import (
+    compare_stdout_results as compare_stdout_results,
+)
+from verifiers.v1.utils.score import (
+    extract_boxed_answer as extract_boxed_answer,
+)
+from verifiers.v1.utils.score import (
+    parse_judge_choice as parse_judge_choice,
+)
+from verifiers.v1.utils.score import (
+    parse_pytest_outcomes as parse_pytest_outcomes,
+)
+from verifiers.v1.utils.score import (
+    read_answer_file_or_last_reply as read_answer_file_or_last_reply,
+)
+from verifiers.v1.utils.score import (
+    verify_boxed_math_answer as verify_boxed_math_answer,
+)
 
 __all__ = [  # noqa: RUF022 - grouped by public API area
     # types

verifiers/v1/agent.py

@@ -24,7 +24,6 @@
 from verifiers.v1.harness import Harness
 from verifiers.v1.interception import Interception, InterceptionServer
 from verifiers.v1.mcp import SharedToolServer
-from verifiers.v1.retries import backoff, trace_should_retry
 from verifiers.v1.rollout import RolloutRun, _as_messages
 from verifiers.v1.runtimes import (
     NetworkPolicyConfig,
@@ -49,6 +48,7 @@
     resolve_runtime_config,
     validate_pairing,
 )
+from verifiers.v1.utils.retries import backoff, trace_should_retry
 
 __all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"]
 
@@ -241,7 +241,7 @@ def __init__(
         client: Client | None = None,
         interception: Interception | None = None,
     ) -> None:
-        from verifiers.v1.loaders import harness_config_type, load_harness
+        from verifiers.v1.utils.loaders import harness_config_type, load_harness
 
         if config.model is None:
             raise ValueError(

verifiers/v1/cli/dashboard/eval.py

@@ -29,7 +29,7 @@
 from verifiers.v1.utils.interrupt import cleaning_up
 
 if TYPE_CHECKING:
-    from verifiers.v1.push import PushState
+    from verifiers.v1.utils.platform import PushState
 
 # For sizing pages to the terminal: detects the real terminal height/width each access (the live
 # view writes to the same terminal). Reused so we don't rebuild it every refresh tick.
@@ -160,7 +160,7 @@ def _warning(config: EvalConfig) -> Text | None:
     """A local-runtime caution when any code-running seat resolves to the subprocess
     runtime (the tool-less chat loops are exempt), shown above the overview rather
     than as a row in it."""
-    from verifiers.v1.loaders import harness_class
+    from verifiers.v1.utils.loaders import harness_class
 
     if any(
         getattr(config.env, role).runtime.type == "subprocess"

verifiers/v1/cli/debug.py

@@ -24,12 +24,12 @@
     with_positional_taskset,
 )
 from verifiers.v1.configs.cli.debug import DebugConfig
-from verifiers.v1.decorators import invoke
 from verifiers.v1.runtimes import ProgramResult, Runtime, make_runtime
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import AgentInfo, Error, Trace, TraceTask
 from verifiers.v1.utils.compile import resolve_runtime_config
+from verifiers.v1.utils.decorators import invoke
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
 

verifiers/v1/cli/eval/main.py

@@ -120,7 +120,7 @@ def main(argv: list[str] | None = None) -> None:
         # disk. Exit on the conventional Ctrl-C code without a traceback.
         raise SystemExit(130)
     if config.push and not rich:
-        from verifiers.v1.push import push_traces
+        from verifiers.v1.utils.platform import push_traces
 
         push_traces(episodes, config)
     if not rich:  # --rich is the whole output; otherwise dump each trace as JSON

verifiers/v1/cli/eval/runner.py

@@ -3,6 +3,7 @@
 import asyncio
 import contextlib
 import logging
+import random
 import time
 
 from verifiers.v1.cli.dashboard import dashboard
@@ -17,8 +18,8 @@
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.env import Env, RunSlot
 from verifiers.v1.episode import Episode
+from verifiers.v1.taskset import SEED
 from verifiers.v1.trace import EvalRunInfo
-from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
 
@@ -86,7 +87,7 @@ async def on_complete(episode: Episode) -> None:
         slots = [RunSlot.finished(episode) for episode in finished] + planned
         push_state = None
         if config.push and config.rich:
-            from verifiers.v1.push import PushState
+            from verifiers.v1.utils.platform import PushState
 
             push_state = PushState()
         display = (
@@ -102,7 +103,7 @@ async def on_complete(episode: Episode) -> None:
             if (
                 push_state is not None
             ):  # upload off the event loop so the view keeps refreshing
-                from verifiers.v1.push import push_traces
+                from verifiers.v1.utils.platform import push_traces
 
                 push_state.started = True
                 await asyncio.to_thread(push_traces, episodes, config, push_state)
@@ -136,7 +137,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
     )
     tasks = []
     if not legacy:
-        from verifiers.v1.loaders import load_taskset
+        from verifiers.v1.utils.loaders import load_taskset
 
         # The client owns the taskset: load it here, once — the server (and its pool
         # workers) never load data, they rebuild each dispatched task from its request.
@@ -184,7 +185,10 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
         if legacy:
             info = await client.info()
             group_scored = info.requires_group_scoring
-            idxs = sample(list(range(info.num_tasks)), config.shuffle, config.num_tasks)
+            idxs = list(range(info.num_tasks))
+            if config.shuffle:
+                random.Random(SEED).shuffle(idxs)
+            idxs = idxs[: config.num_tasks]
             plan = [({"task_idx": idx}, config.num_rollouts) for idx in idxs]
         else:
             group_scored = False

verifiers/v1/cli/validate.py

@@ -20,12 +20,12 @@
     with_positional_taskset,
 )
 from verifiers.v1.configs.cli.validate import ValidateConfig
-from verifiers.v1.decorators import invoke
 from verifiers.v1.runtimes import make_runtime
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import Trace, TraceTask
 from verifiers.v1.utils.compile import resolve_runtime_config
+from verifiers.v1.utils.decorators import invoke
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
 

verifiers/v1/configs/agent.py

@@ -57,7 +57,10 @@ def _resolve_harness(cls, data):
         stays None = the taskset's default). The lazy import keeps class-body
         `AgentConfig()` defaults constructible while this module initializes."""
         if isinstance(data, dict) and data.get("harness") is not None:
-            from verifiers.v1.loaders import harness_config_type, narrow_plugin_field
+            from verifiers.v1.utils.loaders import (
+                harness_config_type,
+                narrow_plugin_field,
+            )
 
             narrow_plugin_field(data, "harness", harness_config_type, "bash")
         return data

verifiers/v1/configs/cli/debug.py

@@ -52,7 +52,7 @@ def name(self) -> str:
     @model_validator(mode="before")
     @classmethod
     def resolve_taskset(cls, data):
-        from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
+        from verifiers.v1.utils.loaders import narrow_plugin_field, taskset_config_type
 
         narrow_plugin_field(data, "taskset", taskset_config_type)
         return data

verifiers/v1/configs/cli/env.py

@@ -35,7 +35,7 @@ def resolve_env_field(data: dict, narrowed: "type[EnvConfig] | None" = None) ->
                     raw.model_dump() if isinstance(raw, BaseConfig) else raw
                 )
             return data
-        from verifiers.v1.loaders import resolve_env_config
+        from verifiers.v1.utils.loaders import resolve_env_config
 
         data["env"] = resolve_env_config(raw)
     except ValidationError as e:

verifiers/v1/configs/cli/replay.py

@@ -50,7 +50,7 @@ def name(self) -> str:
     @model_validator(mode="before")
     @classmethod
     def _resolve_taskset(cls, data):
-        from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
+        from verifiers.v1.utils.loaders import narrow_plugin_field, taskset_config_type
 
         if isinstance(data, dict) and not data.get("taskset"):
             # The base layer is usually a saved eval config, which keeps its

verifiers/v1/configs/cli/validate.py

@@ -54,7 +54,7 @@ def _validate_only(self):
     @model_validator(mode="before")
     @classmethod
     def _resolve_taskset(cls, data):
-        from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
+        from verifiers.v1.utils.loaders import narrow_plugin_field, taskset_config_type
 
         narrow_plugin_field(data, "taskset", taskset_config_type)
         return data

verifiers/v1/configs/env.py

@@ -100,7 +100,10 @@ def _resolve_taskset(cls, data):
         """Narrow `taskset` to its concrete config type by `id`; lazy import for
         the same reason as `AgentConfig._resolve_harness`."""
         if isinstance(data, dict) and data.get("taskset") is not None:
-            from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
+            from verifiers.v1.utils.loaders import (
+                narrow_plugin_field,
+                taskset_config_type,
+            )
 
             narrow_plugin_field(data, "taskset", taskset_config_type)
         return data
@@ -157,7 +160,7 @@ def _declared_agent_configs(config: EnvConfig) -> dict[str, AgentConfig]:
 def default_agent_harness(taskset_id: str) -> HarnessConfig:
     """What an unpinned role's `harness=None` resolves to: the taskset's bundled
     harness when it ships one, else the built-in `bash`."""
-    from verifiers.v1.loaders import default_harness_id, harness_config_type
+    from verifiers.v1.utils.loaders import default_harness_id, harness_config_type
 
     ident = default_harness_id(taskset_id)
     return harness_config_type(ident).model_validate({"id": ident})

verifiers/v1/configs/judge.py

@@ -33,7 +33,7 @@ def judge_key(config: JudgeConfig) -> str:
 
 
 def resolve_judges(entries: Sequence[Any]) -> list[JudgeConfig]:
-    from verifiers.v1.loaders import judge_config_type
+    from verifiers.v1.utils.loaders import judge_config_type
 
     resolved = []
     for entry in entries:

verifiers/v1/env.py

@@ -29,12 +29,12 @@
     requires_tunnel,
 )
 from verifiers.v1.mcp import SharedToolServer, serve_shared
-from verifiers.v1.retries import run_episode_with_retry
 from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local
 from verifiers.v1.task import Task
 from verifiers.v1.trace import Error, Trace
 from verifiers.v1.utils.generic import concrete_type
 from verifiers.v1.utils.memory import trim_memory_periodically
+from verifiers.v1.utils.retries import run_episode_with_retry
 
 logger = logging.getLogger(__name__)
 
@@ -81,7 +81,7 @@ class Env(ABC, Generic[ConfigT]):
     task the agent actually receives — an env-minted task carries its own needs."""
 
     def __init__(self, config: ConfigT) -> None:
-        from verifiers.v1.loaders import load_harness, load_taskset
+        from verifiers.v1.utils.loaders import load_harness, load_taskset
 
         config_cls = concrete_type(type(self), EnvConfig, origin=Env) or EnvConfig
         if not isinstance(config, config_cls):

verifiers/v1/harness.py

@@ -8,11 +8,11 @@
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
-from verifiers.v1.decorators import discover_decorated, invoke_all
 from verifiers.v1.errors import HarnessError, SandboxError, boundary
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.types import Messages
+from verifiers.v1.utils.decorators import discover_decorated, invoke_all
 
 if TYPE_CHECKING:
     # Annotation-only: `Trace` appears in signatures only, so this module stays