diff --git a/examples/harnesses/compact/compact/harness.py b/examples/harnesses/compact/compact/harness.py index 2ff2e8320d..5f121715b2 100644 --- a/examples/harnesses/compact/compact/harness.py +++ b/examples/harnesses/compact/compact/harness.py @@ -27,6 +27,8 @@ class CompactingHarnessConfig(HarnessConfig): class CompactingHarness(Harness[CompactingHarnessConfig]): + SUPPORTS_TASK_TOOLS = True + async def launch( self, ctx: RolloutContext, diff --git a/packages/harnesses/harnesses/rlm/__init__.py b/packages/harnesses/harnesses/rlm/__init__.py index 976a9de982..2c9c78df1d 100644 --- a/packages/harnesses/harnesses/rlm/__init__.py +++ b/packages/harnesses/harnesses/rlm/__init__.py @@ -35,9 +35,8 @@ class RLMHarnessConfig(HarnessConfig): class RLMHarness(Harness[RLMHarnessConfig]): - APPENDS_SYSTEM_PROMPT = ( - True # rlm appends it to its prompt via RLM_APPEND_TO_SYSTEM_PROMPT - ) + APPENDS_SYSTEM_PROMPT = True + SUPPORTS_TASK_TOOLS = False async def launch( self, diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index cc248efc16..385ca63622 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -124,10 +124,20 @@ def _resolve_plugins(cls, data): class Environment: def __init__(self, config: EnvConfig) -> None: from verifiers.v1.loaders import load_harness, load_taskset + from verifiers.v1.taskset import Taskset self.config = config self.taskset = load_taskset(config.taskset) self.harness = load_harness(config.harness) + if ( + not self.harness.SUPPORTS_TASK_TOOLS + and type(self.taskset).tools is not Taskset.tools + ): + raise ValueError( + f"Harness {self.harness.config.id!r} does not support task tools, but taskset " + f"{self.taskset.config.id!r} exposes tool servers (MCP). Run it with a harness " + f"that supports task tools (e.g. --harness.id default), or use a taskset without tools." + ) self.harness_timeout = config.timeout.rollout self.scoring_timeout = config.timeout.scoring self.limits = RolloutLimits( diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index f2710d19bb..eefc4328a1 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -55,6 +55,9 @@ class Harness(ABC, Generic[ConfigT]): APPENDS_SYSTEM_PROMPT: ClassVar[bool] = ( False # emit task.system_prompt as a system message (else fold into the user message) ) + SUPPORTS_TASK_TOOLS: ClassVar[bool] = ( + True # expose a task's MCP tool servers to the model; set False for harnesses without an MCP client + ) def __init__(self, config: ConfigT) -> None: self.config = config