diff --git a/tests/v1/conftest.py b/tests/v1/conftest.py index bd3fa1a10a..6fe560ff75 100644 --- a/tests/v1/conftest.py +++ b/tests/v1/conftest.py @@ -164,7 +164,10 @@ def _eval_config( seat_cfg.setdefault("max_output_tokens", max_tokens) seat_cfg.setdefault("timeout", {"rollout": rollout_timeout, "scoring": 60}) # Flake resilience: retries are per-agent now (flat RetryConfig). - seat_cfg.setdefault("retries", {"max_retries": 2, "include": ["ProviderError"]}) + seat_cfg.setdefault( + "retries", + {"max_retries": 2, "include": ["ProviderError", "HarnessError"]}, + ) return EvalConfig( env={ "taskset": taskset_cfg, diff --git a/tests/v1/fixtures/echo_tool_v1.py b/tests/v1/fixtures/echo_tool_v1.py index ea7ca6cd3a..d59f561b70 100644 --- a/tests/v1/fixtures/echo_tool_v1.py +++ b/tests/v1/fixtures/echo_tool_v1.py @@ -11,6 +11,7 @@ """ import verifiers.v1 as vf +from verifiers.v1.types import content_text PHRASE = "hello world" ECHO_TOKEN = "ok-7f3" # the tool stamps this; only a real tool call can surface it @@ -36,11 +37,9 @@ def toolsets(cls, config: EchoToolTaskConfig) -> list[vf.Toolset]: @vf.reward(weight=1.0) async def echoed(self, trace: vf.Trace) -> float: - # The stamped token surfaces in an ASSISTANT message only if the model called - # the MCP tool and relayed its result — wherever in the exchange that happened - # (in a conversation the last turn may be a closing pleasantry). - replies = ((m.content or "").lower() for m in trace.assistant_messages) - return float(any(PHRASE in r and ECHO_TOKEN in r for r in replies)) + # A stamped TOOL result proves the tool really ran with the phrase. + results = (content_text(m.content).lower() for m in trace.tool_messages) + return float(any(PHRASE in r and ECHO_TOKEN in r for r in results)) class EchoToolConfig(vf.TasksetConfig): diff --git a/verifiers/v1/dialects/chat.py b/verifiers/v1/dialects/chat.py index 23277f4c4f..5fcdc70c59 100644 --- a/verifiers/v1/dialects/chat.py +++ b/verifiers/v1/dialects/chat.py @@ -141,7 +141,11 @@ def _content_to_wire(content): def message_to_wire(message: Message) -> dict: if message.role == "assistant": - wire: dict = {"role": "assistant", "content": message.content} + # Strict providers reject `content: null` without tool calls. + content = message.content + if content is None and not message.tool_calls: + content = "" + wire: dict = {"role": "assistant", "content": content} if message.provider_state: wire["reasoning_details"] = message.provider_state elif message.reasoning_content is not None: diff --git a/verifiers/v1/envs/user_sim/env.py b/verifiers/v1/envs/user_sim/env.py index 6f2f08bf10..1dd2ce8e66 100644 --- a/verifiers/v1/envs/user_sim/env.py +++ b/verifiers/v1/envs/user_sim/env.py @@ -21,13 +21,14 @@ import verifiers.v1 as vf -PERSONA = """You are role-playing a USER talking to an AI assistant. This is your situation and goal: +PERSONA = """You are role-playing a USER talking to an AI assistant. This is your situation — what you want the ASSISTANT to do for you: {scenario} Rules: - Open the conversation with your request, in your own words. - Stay in character: short, natural user messages; never act as the assistant. +- The assistant does the work, not you: never perform the task yourself — any tool call, answer, or output format it asks for must come from the assistant; ask for it instead. - Reveal details only when asked, as a real user would. - When the assistant has fully met your goal — or you are convinced it cannot — reply with exactly {done} and nothing else.""" diff --git a/verifiers/v1/harnesses/openclaw/harness.py b/verifiers/v1/harnesses/openclaw/harness.py index 957f4deefb..470413849d 100644 --- a/verifiers/v1/harnesses/openclaw/harness.py +++ b/verifiers/v1/harnesses/openclaw/harness.py @@ -117,10 +117,8 @@ async def launch( data: TaskData, ) -> ProgramResult: system_prompt, prompt = self.resolve_prompt(data) - reasoning = ctx.sampling.reasoning_effort not in ( - None, - "none", - ) or ctx.model.rsplit("/", 1)[-1].startswith(("gpt-5", "o1", "o3", "o4")) + # Opt-in via sampling: OpenClaw redacts persisted encrypted reasoning, so resumed sessions 400 replaying it. + reasoning = ctx.sampling.reasoning_effort not in (None, "none") directory = OPENCLAW_DIR.format(version=self.config.version) state_dir = f".vf-openclaw/{trace.id}" config_path = f"{state_dir}/openclaw.json"