fix: gate task-tool tasksets on harness support (SUPPORTS_TASK_TOOLS)

examples/harnesses/compact/compact/harness.py

@@ -27,6 +27,8 @@ class CompactingHarnessConfig(HarnessConfig):
 
 
 class CompactingHarness(Harness[CompactingHarnessConfig]):
+    SUPPORTS_TASK_TOOLS = True
+
     async def launch(
         self,
         ctx: RolloutContext,

packages/harnesses/harnesses/rlm/__init__.py

@@ -35,9 +35,8 @@ class RLMHarnessConfig(HarnessConfig):
 
 
 class RLMHarness(Harness[RLMHarnessConfig]):
-    APPENDS_SYSTEM_PROMPT = (
-        True  # rlm appends it to its prompt via RLM_APPEND_TO_SYSTEM_PROMPT
-    )
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_TASK_TOOLS = False
 
     async def launch(
         self,

verifiers/v1/env.py

@@ -124,10 +124,20 @@ def _resolve_plugins(cls, data):
 class Environment:
     def __init__(self, config: EnvConfig) -> None:
         from verifiers.v1.loaders import load_harness, load_taskset
+        from verifiers.v1.taskset import Taskset
 
         self.config = config
         self.taskset = load_taskset(config.taskset)
         self.harness = load_harness(config.harness)
+        if (
+            not self.harness.SUPPORTS_TASK_TOOLS
+            and type(self.taskset).tools is not Taskset.tools
+        ):
+            raise ValueError(
+                f"Harness {self.harness.config.id!r} does not support task tools, but taskset "
+                f"{self.taskset.config.id!r} exposes tool servers (MCP). Run it with a harness "
+                f"that supports task tools (e.g. --harness.id default), or use a taskset without tools."
+            )
         self.harness_timeout = config.timeout.rollout
         self.scoring_timeout = config.timeout.scoring
         self.limits = RolloutLimits(

verifiers/v1/harness.py

@@ -55,6 +55,9 @@ class Harness(ABC, Generic[ConfigT]):
     APPENDS_SYSTEM_PROMPT: ClassVar[bool] = (
         False  # emit task.system_prompt as a system message (else fold into the user message)
     )
+    SUPPORTS_TASK_TOOLS: ClassVar[bool] = (
+        True  # expose a task's MCP tool servers to the model; set False for harnesses without an MCP client
+    )
 
     def __init__(self, config: ConfigT) -> None:
         self.config = config