Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 173 additions & 0 deletions examples/tasksets/color_codeword_v1/color_codeword_v1/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
"""color-codeword-v1 — a multi-turn VLM codeword-decoding task (the v1 port of `color-codeword`).

Each turn shows colored squares that map to letters (Red=A, Green=B, ...); the model accumulates
the codeword across turns and, on the final turn, outputs the whole thing. Turn 0's squares are
seeded in the task's `instruction` (a `Messages` prompt carrying images); the later turns are
injected by a colocated `vf.User` (see `user.py`) the interception server drives after each
assistant turn. Reward is an exact match of the final codeword; a partial-match metric tracks
per-position accuracy. Images carry through the v1 message graph as `mm_kwargs` for training.
"""

import base64
import json
import random
import re
import sys
from io import BytesIO

from PIL import Image

import verifiers.v1 as vf

COLOR_MAP = {
"red": "A",
"green": "B",
"blue": "C",
"yellow": "D",
"purple": "E",
"cyan": "F",
"orange": "G",
"white": "H",
"black": "I",
}
COLOR_RGB = {
"red": (255, 0, 0),
"green": (0, 255, 0),
"blue": (0, 0, 255),
"yellow": (255, 255, 0),
"purple": (128, 0, 128),
"cyan": (0, 255, 255),
"orange": (255, 165, 0),
"white": (255, 255, 255),
"black": (0, 0, 0),
}

SYSTEM_PROMPT = """You will be shown colored squares across multiple turns. Each color maps to a letter:

Red=A, Green=B, Blue=C, Yellow=D, Purple=E, Cyan=F, Orange=G, White=H, Black=I

Example: Turn 1 shows Red, Blue. Turn 2 shows Green, Yellow. The full codeword is "ACBD" (all 4 letters in order).

After each turn, output your accumulated codeword so far. Output ONLY the letters with NO spaces."""


def color_data_url(color: str, size: int = 100) -> str:
"""A solid-color PNG square as a base64 `data:` URL."""
img = Image.new("RGB", (size, size), COLOR_RGB[color])
buf = BytesIO()
img.save(buf, format="PNG")
return f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode()}"


def image_content(colors: list[str], text: str) -> list[dict]:
"""An OpenAI content list — one image part per color, then the text. Used by the user
simulator's later turns (the interception server re-types it, preserving the images)."""
return [
{"type": "image_url", "image_url": {"url": color_data_url(c)}} for c in colors
] + [{"type": "text", "text": text}]


def turn_text(turn: int, count: int, max_turns: int, total: int) -> str:
"""The user text shown alongside a turn's squares (mirrors the v0 wording)."""
if turn == 0:
return f"Here are {count} squares."
if turn == max_turns - 1:
return (
f"Here are {count} more squares. Combine your previous answer with these new "
f"letters to output all {total} letters."
)
return f"Here are {count} more squares."


def extract_codeword(text: str) -> str:
"""The longest standalone A–I run in `text` (case-insensitive); else all A–I characters."""
text = text.upper()
matches = re.findall(r"\b[A-I]+\b", text)
return (
max(matches, key=len)
if matches
else "".join(c for c in text if c in "ABCDEFGHI")
)


class ColorCodewordConfig(vf.TasksetConfig):
num_examples: int = 1000
"""Number of synthetic episodes to generate."""
images_per_turn: int = 2
"""Colored squares shown per turn."""
max_turns: int = 3
"""Turns per episode; the codeword has `images_per_turn * max_turns` letters."""
seed: int = 42
"""RNG seed for reproducible color sequences."""


class ColorCodewordTask(vf.Task):
answer: str
"""The full expected codeword (one letter per square shown, in order)."""
info: dict
"""The episode the user simulator replays: `colors_per_turn` and `max_turns`."""


class ColorCodewordTaskset(vf.Taskset[ColorCodewordTask, ColorCodewordConfig]):
def load_tasks(self) -> list[ColorCodewordTask]:
c = self.config
assert c.images_per_turn >= 1 and c.max_turns >= 1, (
"need images_per_turn >= 1 and max_turns >= 1"
)
rng = random.Random(c.seed)
colors = list(COLOR_MAP)
length = c.images_per_turn * c.max_turns
tasks: list[ColorCodewordTask] = []
for idx in range(c.num_examples):
sequence = [rng.choice(colors) for _ in range(length)]
answer = "".join(COLOR_MAP[col] for col in sequence)
colors_per_turn = [
sequence[t * c.images_per_turn : (t + 1) * c.images_per_turn]
for t in range(c.max_turns)
]
turn0 = colors_per_turn[0]
text = turn_text(0, len(turn0), c.max_turns, len(turn0))
parts = [
vf.ImageUrlContentPart(
image_url=vf.ImageUrlSource(url=color_data_url(col))
)
for col in turn0
] + [vf.TextContentPart(text=text)]
tasks.append(
ColorCodewordTask(
idx=idx,
instruction=[vf.UserMessage(content=parts)],
system_prompt=SYSTEM_PROMPT,
answer=answer,
info={"colors_per_turn": colors_per_turn, "max_turns": c.max_turns},
)
)
return tasks

def user(self, task: ColorCodewordTask) -> vf.User:
return vf.User(
name="user",
command=[sys.executable, "-m", "color_codeword_v1.user"],
env={"COLOR_CODEWORD_INFO": json.dumps(task.info)},
)

@vf.reward(weight=1.0)
async def exact_match(self, task: ColorCodewordTask, trace: vf.Trace) -> float:
responses = trace.assistant_messages
last = (responses[-1].content if responses else "") or ""
return 1.0 if extract_codeword(last) == task.answer else 0.0

@vf.metric
async def partial_match(self, task: ColorCodewordTask, trace: vf.Trace) -> float:
if not task.answer:
return 0.0
responses = trace.assistant_messages
last = (responses[-1].content if responses else "") or ""
extracted = extract_codeword(last)
return sum(1 for a, b in zip(task.answer, extracted) if a == b) / len(
task.answer
)


def load_taskset(config: ColorCodewordConfig) -> ColorCodewordTaskset:
return ColorCodewordTaskset(config)
46 changes: 46 additions & 0 deletions examples/tasksets/color_codeword_v1/color_codeword_v1/user.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
"""color-codeword-v1 user simulator: reveals each turn's colored squares after the prior answer.

Launched per rollout as a `vf.User` subprocess. Holds the episode's per-turn colors (from
`COLOR_CODEWORD_INFO`) and serves a single `respond` tool: after each assistant turn it injects the
next turn's squares as a user message (image_url parts), until every `max_turns` turn has been
answered. This colocates v0's per-turn image presentation with the agent — the framework drives it,
never the model.
"""

import json
import os

from mcp.server.fastmcp import FastMCP

import verifiers.v1 as vf

from color_codeword_v1 import image_content, turn_text

INFO = json.loads(os.environ["COLOR_CODEWORD_INFO"])
COLORS_PER_TURN = INFO["colors_per_turn"]
MAX_TURNS = INFO["max_turns"]

mcp = FastMCP("user")

_turns = 0 # assistant turns taken so far (one respond call per turn)


@mcp.tool()
def respond(message: str) -> str:
"""Show the next turn's squares, or end the episode once every turn has been answered."""
global _turns
_turns += 1
if _turns >= MAX_TURNS:
return json.dumps({"messages": [], "done": True})
colors = COLORS_PER_TURN[_turns]
total = sum(len(COLORS_PER_TURN[t]) for t in range(_turns + 1))
text = turn_text(_turns, len(colors), MAX_TURNS, total)
return json.dumps(
{
"messages": [{"role": "user", "content": image_content(colors, text)}],
"done": False,
}
)


vf.run_mcp_server(mcp)
13 changes: 13 additions & 0 deletions examples/tasksets/color_codeword_v1/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[project]
name = "color-codeword-v1"
version = "0.1.0"
description = "color-codeword-v1 — a multi-turn VLM codeword-decoding task, driven by a colocated user simulator."
requires-python = ">=3.10"
dependencies = ["pillow"]

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.hatch.build.targets.wheel]
packages = ["color_codeword_v1"]
15 changes: 14 additions & 1 deletion packages/harnesses/harnesses/default/harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from verifiers.v1.harness import Harness, HarnessConfig
from verifiers.v1.clients import RolloutContext
from verifiers.v1.clients.openai import message_to_wire
from verifiers.v1.runtimes import ProgramResult, Runtime
from verifiers.v1.trace import Trace

Expand All @@ -36,6 +37,9 @@ class DefaultHarnessConfig(HarnessConfig):
class DefaultHarness(Harness[DefaultHarnessConfig]):
APPENDS_SYSTEM_PROMPT = True # program.py emits it as a real system message
SUPPORTS_USER_SIM = True # the chat loop drives a task's user simulator
SUPPORTS_MESSAGE_INSTRUCTION = (
True # seeds the chat loop from a Messages instruction (e.g. images)
)

async def launch(
self,
Expand Down Expand Up @@ -64,4 +68,13 @@ async def launch(
env["MCP_CONFIG"] = json.dumps(
{"mcpServers": {name: {"url": url} for name, url in mcp_urls.items()}}
)
return await runtime.run_uv_script(PROGRAM_SOURCE, args=[instruction], env=env)
# A Messages instruction (e.g. an image-bearing prompt) seeds the chat loop directly;
# a plain string is the single first user message.
if isinstance(instruction, str):
args = [instruction]
else:
env["INITIAL_MESSAGES"] = json.dumps(
[message_to_wire(m) for m in instruction]
)
args = [""]

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Stale INITIAL_MESSAGES env leaks

Medium Severity

For string instructions, DefaultHarness never sets INITIAL_MESSAGES, while program.py treats any non-empty parsed value as the chat seed. On the subprocess runtime, the child inherits the host os.environ, so a leftover INITIAL_MESSAGES from an earlier multimodal run can override the argv user prompt.

Additional Locations (1)
Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit 15ee62a. Configure here.

return await runtime.run_uv_script(PROGRAM_SOURCE, args=args, env=env)
8 changes: 7 additions & 1 deletion packages/harnesses/harnesses/default/program.py
Comment thread
macroscopeapp[bot] marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,13 @@ async def main() -> None:
messages = (
[{"role": "system", "content": system_prompt}] if system_prompt else []
)
messages.append({"role": "user", "content": instruction})
# A Messages instruction (e.g. an image-bearing prompt) arrives pre-built as OpenAI
# wire dicts; otherwise the single argv string is the first user message.
initial = json.loads(os.environ.get("INITIAL_MESSAGES", "[]"))
if initial:
messages.extend(initial)
else:
messages.append({"role": "user", "content": instruction})
while True:
message = await chat(messages, tools)
messages.append(message.model_dump(exclude_none=True))
Expand Down
10 changes: 10 additions & 0 deletions verifiers/v1/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,17 @@
)
from verifiers.v1.types import (
AssistantMessage,
ContentPart,
ImageUrlContentPart,
ImageUrlSource,
Message,
MessageContent,
Messages,
Response,
SamplingConfig,
StrictBaseModel,
SystemMessage,
TextContentPart,
Tool,
ToolCall,
TurnTokens,
Expand All @@ -74,12 +79,17 @@
"ensure_installed",
"env_name",
"AssistantMessage",
"ContentPart",
"ImageUrlContentPart",
"ImageUrlSource",
"Message",
"MessageContent",
"Messages",
"Response",
"SamplingConfig",
"StrictBaseModel",
"SystemMessage",
"TextContentPart",
"Tool",
"ToolCall",
"ToolMessage",
Expand Down
10 changes: 9 additions & 1 deletion verifiers/v1/clients/openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,14 @@
FINISH_REASONS = frozenset({"stop", "length", "tool_calls"})


def _content_to_wire(content):
"""Plain text passes through; a content-part list becomes OpenAI wire dicts (so the
provider / renderer sees the native `image_url` shape)."""
if isinstance(content, str):
return content
return [part.model_dump() for part in content]


def message_to_wire(message: Message) -> dict:
if message.role == "assistant":
wire: dict = {"role": "assistant", "content": message.content}
Expand All @@ -47,7 +55,7 @@ def message_to_wire(message: Message) -> dict:
"tool_call_id": message.tool_call_id,
"content": message.content,
}
return {"role": message.role, "content": message.content}
return {"role": message.role, "content": _content_to_wire(message.content)}


def tool_to_wire(tool: Tool) -> dict:
Expand Down
1 change: 1 addition & 0 deletions verifiers/v1/clients/renderer.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ def response_from_generate(result: dict, model: str) -> Response:
completion_ids=completion_ids,
completion_logprobs=result.get("completion_logprobs") or [],
message_spans=message_spans,
multi_modal_data=result.get("multi_modal_data"),
),
)

Expand Down
Loading
Loading