From 318dbadec18e431b268e678cdb5e05e10af8056b Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Wed, 10 Jun 2026 23:56:26 +0000 Subject: [PATCH 1/6] feat(v1): add scaleswe taskset + per-task setup/workdir hooks scaleswe-v1 ports the v0 ComposableEnv Scale-SWE taskset to v1: each row carries its per-task image + workdir, runs its pre_commands in setup() before the agent, and scores with a single `solved` reward that restores the test files to base, applies the f2p test, and runs the merged F2P+P2P pytest ids through a self-contained scorer (1.0 iff every expected id passes). Two small, general framework hooks enable it: - Task.workdir, injected into the runtime config (symmetric with Task.image), so the agent and scoring run in the row's repo dir. - Taskset.setup(task, runtime), run by the rollout after runtime.start() and before the harness, for per-task runtime prep. Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/tasksets/scaleswe_v1/_scorer.py | 86 ++++++++++ examples/tasksets/scaleswe_v1/pyproject.toml | 13 ++ examples/tasksets/scaleswe_v1/scaleswe_v1.py | 163 +++++++++++++++++++ verifiers/v1/env.py | 2 + verifiers/v1/rollout.py | 1 + verifiers/v1/task.py | 5 + verifiers/v1/taskset.py | 6 + 7 files changed, 276 insertions(+) create mode 100644 examples/tasksets/scaleswe_v1/_scorer.py create mode 100644 examples/tasksets/scaleswe_v1/pyproject.toml create mode 100644 examples/tasksets/scaleswe_v1/scaleswe_v1.py diff --git a/examples/tasksets/scaleswe_v1/_scorer.py b/examples/tasksets/scaleswe_v1/_scorer.py new file mode 100644 index 0000000000..6f9d41aa0f --- /dev/null +++ b/examples/tasksets/scaleswe_v1/_scorer.py @@ -0,0 +1,86 @@ +"""Run the merged F2P+P2P pytest ids and print 1.0 iff every expected id passed. + +Run inside the task's repo (cwd) by the testbed python (which has pytest + the project +installed) — NOT a uv script, since the tests need the project's own environment. argv[1] +is a JSON list of pytest node ids. We write JUnit XML and match each expected id against +it (a pytest node id and the JUnit classname/name don't line up, so we try a few forms), +printing the score on the last line. Any failure prints 0.0. +""" + +import json +import re +import sys +import xml.etree.ElementTree as ET + +import pytest + +XML = "/tmp/scaleswe_results.xml" + + +def _normalize(value: str) -> str: + parts = value.strip().split("::") + if parts and parts[0].endswith(".py"): + parts[0] = parts[0][:-3] + return ".".join(parts).replace("/", ".").strip(".") + + +def _all_passed(xml_content: str, expected: list[str]) -> bool: + try: + root = ET.fromstring(xml_content) + except ET.ParseError: + return False + exact = set(expected) + norm = {_normalize(t): t for t in expected} + fp = {re.sub(r"\s+", "", _normalize(t)): t for t in expected} + matched: dict[str, str] = {} + found: set[str] = set() + for tc in root.iter("testcase"): + if tc.find("skipped") is not None: + continue + name, classname = tc.get("name", ""), tc.get("classname", "") + file_attr = tc.get("file", "") + status = ( + "failed" + if tc.find("failure") is not None or tc.find("error") is not None + else "passed" + ) + for candidate in ( + f"{file_attr}::{name}" if file_attr else "", + _normalize(f"{classname}.{name}"), + re.sub(r"\s+", "", _normalize(f"{classname}.{name}")), + f"{classname.replace('.', '/')}.py::{name}", + ): + original = ( + candidate + if candidate in exact + else norm.get(candidate) or fp.get(candidate) + ) + if original: + matched[original] = status + found.add(original) + break + return ( + bool(found) + and all(status == "passed" for status in matched.values()) + and not [t for t in expected if t not in found] + ) + + +def main() -> None: + expected = json.loads(sys.argv[1]) + if not expected: + print(0.0) + return + pytest.main( + ["-vv", f"--junitxml={XML}", "-o", "addopts=", "--rootdir=.", *expected] + ) + try: + xml_content = open(XML).read() + except OSError: + print(0.0) + return + print(1.0 if _all_passed(xml_content, expected) else 0.0) + + +if __name__ == "__main__": + main() diff --git a/examples/tasksets/scaleswe_v1/pyproject.toml b/examples/tasksets/scaleswe_v1/pyproject.toml new file mode 100644 index 0000000000..2d26453be1 --- /dev/null +++ b/examples/tasksets/scaleswe_v1/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "scaleswe-v1" +version = "0.1.0" +description = "scaleswe-v1 — Scale-SWE issue-resolving tasks (agentic; sandbox pytest reward)." +requires-python = ">=3.10" +dependencies = ["datasets"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build] +include = ["scaleswe_v1.py", "_scorer.py", "pyproject.toml"] diff --git a/examples/tasksets/scaleswe_v1/scaleswe_v1.py b/examples/tasksets/scaleswe_v1/scaleswe_v1.py new file mode 100644 index 0000000000..bf0d2c90c6 --- /dev/null +++ b/examples/tasksets/scaleswe_v1/scaleswe_v1.py @@ -0,0 +1,163 @@ +"""scaleswe-v1 — Scale-SWE (AweAI-Team/Scale-SWE) as a v1 taskset. + +Each row ships a per-task Docker image with the repo checked out, `pre_commands` that +reset it to the base commit on a clean `scaleswe` branch, F2P/P2P pytest ids, and an +optional `f2p_patch` / `f2p_script` carrying the failing test. `setup` runs the row's +`pre_commands` in the live runtime before the agent (the runtime's workdir is the row's +repo). The `solved` reward restores the test files to base (the agent only fixes the +source), applies the f2p test, then runs the merged F2P+P2P ids through `_scorer.py` — +which scores 1.0 iff every expected id passed. A v1 port of the v0 ComposableEnv +`ScaleSWETaskSet`. +""" + +import json +from pathlib import Path + +import verifiers.v1 as vf + +REGISTRY = "us-central1-docker.pkg.dev/prime-intellect-platform/prod-sandbox" + +# The testbed conda env (with the project + pytest installed) and quiet, non-interactive +# tooling — exported for every command the taskset runs in the sandbox. +ENV = { + "PATH": ( + "/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/bin:" + "/opt/conda/envs/testbed/bin:/opt/conda/bin:" + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ), + "PAGER": "cat", + "MANPAGER": "cat", + "LESS": "-R", + "PIP_PROGRESS_BAR": "off", + "TQDM_DISABLE": "1", + "CI": "1", +} + +# Restore the test files to the base commit so the agent's edits to the source are scored +# against the original tests (and tests it added are dropped). `$base` comes from the env. +RESTORE = r""" +git checkout "$base" -- tests/ test/ Test/ Tests/ 2>/dev/null || true +git ls-tree -r --name-only "$base" 2>/dev/null | while IFS= read -r path; do + case "$path" in + test_*.py|*/test_*.py|*_test.py|*/*_test.py|conftest.py|*/conftest.py) + git checkout "$base" -- "$path" 2>/dev/null || true ;; + esac +done +git ls-files 2>/dev/null | while IFS= read -r path; do + case "$path" in + tests/*|test/*|Test/*|Tests/*|test_*.py|*/test_*.py|*_test.py|*/*_test.py|conftest.py|*/conftest.py) + if ! git cat-file -e "$base:$path" 2>/dev/null; then + rm -f -- "$path"; git rm -q --cached -- "$path" 2>/dev/null || true + fi ;; + esac +done +""" + +PATCH = "/tmp/scaleswe_f2p.patch" +SCORER = "/tmp/scaleswe_scorer.py" +SCORER_SRC = (Path(__file__).parent / "_scorer.py").read_bytes() + + +def _image(image_url: str) -> str: + return image_url if image_url.startswith(REGISTRY) else f"{REGISTRY}/{image_url}" + + +def _ids(raw: str | list[str] | None) -> list[str]: + """F2P/P2P ids arrive as a list or a JSON-encoded string; normalize to list[str].""" + if not raw: + return [] + if isinstance(raw, list): + return [str(t).strip() for t in raw if t] + raw = raw.strip() + try: + parsed = json.loads(raw) + except (json.JSONDecodeError, TypeError): + return [raw] if raw else [] + if isinstance(parsed, list): + return [str(t).strip() for t in parsed if t] + return [parsed] if isinstance(parsed, str) and parsed else [] + + +class ScaleSWETask(vf.Task): + base_commit: str + """Commit the repo is reset to before the agent runs and tests are scored against.""" + pre_commands: str + """Shell run in the repo before the agent — resets to `base_commit` on branch `scaleswe`.""" + f2p_patch: str = "" + """Optional patch adding the fail-to-pass test (applied before scoring).""" + f2p_script: str = "" + """Optional test file uploaded as `test_fail_to_pass.py` before scoring.""" + fail_to_pass: list[str] = [] + pass_to_pass: list[str] = [] + + +class ScaleSWEConfig(vf.TasksetConfig): + dataset_name: str = "AweAI-Team/Scale-SWE" + split: str = "train" + + +class ScaleSWETaskset(vf.Taskset[ScaleSWETask, ScaleSWEConfig]): + def load_tasks(self) -> list[ScaleSWETask]: + from datasets import load_dataset + + rows = load_dataset(self.config.dataset_name, split=self.config.split) + return [ + ScaleSWETask( + idx=i, + name=row["instance_id"], + instruction=row["problem_statement"], + image=_image(row["image_url"]), + workdir=row["workdir"], + base_commit=row.get("parent_commit") or row.get("base_commit") or "", + pre_commands=(row.get("pre_commands") or "") + .strip() + .removesuffix("\\n"), + f2p_patch=row.get("f2p_patch") or "", + f2p_script=row.get("f2p_script") or "", + fail_to_pass=_ids(row.get("FAIL_TO_PASS")), + pass_to_pass=_ids(row.get("PASS_TO_PASS")), + ) + for i, row in enumerate(rows) + ] + + async def setup(self, task: ScaleSWETask, runtime: vf.Runtime) -> None: + if not task.pre_commands: + raise vf.ProgramError(f"scaleswe row {task.name!r} has no pre_commands") + result = await runtime.run(["sh", "-c", task.pre_commands], ENV) + if result.exit_code != 0: + raise vf.ProgramError( + f"scaleswe setup failed ({task.name}): {result.stderr.strip()[-500:]}" + ) + + @vf.reward(weight=1.0) + async def solved(self, task: ScaleSWETask, runtime: vf.Runtime) -> float: + test_ids = task.fail_to_pass + task.pass_to_pass + if not test_ids: + return 0.0 + await runtime.run(["sh", "-c", RESTORE], {**ENV, "base": task.base_commit}) + if task.f2p_patch.strip(): + await self._apply_f2p_patch(runtime, task.f2p_patch) + if task.f2p_script: + await runtime.write("test_fail_to_pass.py", task.f2p_script.encode()) + await runtime.write(SCORER, SCORER_SRC) + result = await runtime.run(["python", SCORER, json.dumps(test_ids)], ENV) + lines = result.stdout.strip().splitlines() + return float(lines[-1]) if lines else 0.0 + + async def _apply_f2p_patch(self, runtime: vf.Runtime, patch: str) -> None: + # Try strict, then whitespace-tolerant, then a fuzzy `patch`; tolerate a partial + # `--reject` apply last (mirrors the v0 multi-strategy helper). Scoring catches a + # patch that didn't take — a missing test id fails the reward. + await runtime.write(PATCH, patch.encode()) + for cmd in ( + f"git apply --verbose {PATCH}", + f"git apply --verbose --ignore-space-change --ignore-whitespace {PATCH}", + f"patch --batch --fuzz=5 -p1 -i {PATCH}", + f"git apply --verbose --reject --ignore-whitespace {PATCH} || true", + ): + if (await runtime.run(["sh", "-c", cmd], ENV)).exit_code == 0: + return + + +def load_taskset(config: ScaleSWEConfig) -> ScaleSWETaskset: + return ScaleSWETaskset(config) diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index b021cc2ffd..1ad71383d1 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -166,6 +166,8 @@ def runtime_for(self, task: Task) -> RuntimeConfig: "runtime has no container; use the docker or prime runtime" ) updates["image"] = task.image + if task.workdir is not None and "workdir" in type(config).model_fields: + updates["workdir"] = task.workdir for field, value in task.resources.model_dump(exclude_none=True).items(): spec = type(config).model_fields.get(field) if spec is None: diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 1cd0026576..e2d0b15d4c 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -123,6 +123,7 @@ async def run( try: session = RolloutSession(self.ctx, trace, stops, self.limits) await runtime.start() + await self.taskset.setup(self.task, runtime) async with self._serve_interception(interception, runtime, session) as ( endpoint, secret, diff --git a/verifiers/v1/task.py b/verifiers/v1/task.py index 0765a1be75..d4038da125 100644 --- a/verifiers/v1/task.py +++ b/verifiers/v1/task.py @@ -52,6 +52,11 @@ class Task(StrictBaseModel): """Container image this task needs (e.g. its harbor environment). When set, the runtime must be a container (docker/prime): the Environment injects it into the runtime config and refuses the subprocess runtime, which has no container.""" + workdir: str | None = None + """Working directory the harness and scoring run in — the Environment injects it into + the runtime config's `workdir` (where the runtime supports one). For a containerized + task whose image puts the working tree at a non-default path (e.g. a SWE row's + `/workspace/`).""" harness_timeout: float | None = None """Optional per-task harness timeout (seconds). Merges with the eval's `harness_timeout`: cli/toml > this > default (no limit).""" diff --git a/verifiers/v1/taskset.py b/verifiers/v1/taskset.py index 76216882be..c95639e043 100644 --- a/verifiers/v1/taskset.py +++ b/verifiers/v1/taskset.py @@ -109,6 +109,12 @@ def user(self, task: TaskT) -> User | None: multi-turn conversation (e.g. a TextArena game).""" return None + async def setup(self, task: TaskT, runtime: Runtime) -> None: + """Prepare the live runtime for this task, after `runtime.start()` and before the + harness runs. No-op by default; override to run per-task setup in the runtime (e.g. + a SWE row checking out its base commit). Errors propagate and fail the rollout.""" + return None + async def score(self, trace: Trace, runtime: Runtime) -> None: """Score one rollout: run all `@metric` then `@reward` over its trace, concurrently within each phase. Each metric is recorded in `trace.metrics` From d6bfece3b6cf9dd096d9268390b37d023a922477 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Thu, 11 Jun 2026 00:02:50 +0000 Subject: [PATCH 2/6] chore: rename scaleswe _scorer.py -> score.py Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/tasksets/scaleswe_v1/pyproject.toml | 2 +- examples/tasksets/scaleswe_v1/scaleswe_v1.py | 4 ++-- examples/tasksets/scaleswe_v1/{_scorer.py => score.py} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename examples/tasksets/scaleswe_v1/{_scorer.py => score.py} (100%) diff --git a/examples/tasksets/scaleswe_v1/pyproject.toml b/examples/tasksets/scaleswe_v1/pyproject.toml index 2d26453be1..051ad9636a 100644 --- a/examples/tasksets/scaleswe_v1/pyproject.toml +++ b/examples/tasksets/scaleswe_v1/pyproject.toml @@ -10,4 +10,4 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build] -include = ["scaleswe_v1.py", "_scorer.py", "pyproject.toml"] +include = ["scaleswe_v1.py", "score.py", "pyproject.toml"] diff --git a/examples/tasksets/scaleswe_v1/scaleswe_v1.py b/examples/tasksets/scaleswe_v1/scaleswe_v1.py index bf0d2c90c6..ac2c2a4cda 100644 --- a/examples/tasksets/scaleswe_v1/scaleswe_v1.py +++ b/examples/tasksets/scaleswe_v1/scaleswe_v1.py @@ -5,7 +5,7 @@ optional `f2p_patch` / `f2p_script` carrying the failing test. `setup` runs the row's `pre_commands` in the live runtime before the agent (the runtime's workdir is the row's repo). The `solved` reward restores the test files to base (the agent only fixes the -source), applies the f2p test, then runs the merged F2P+P2P ids through `_scorer.py` — +source), applies the f2p test, then runs the merged F2P+P2P ids through `score.py` — which scores 1.0 iff every expected id passed. A v1 port of the v0 ComposableEnv `ScaleSWETaskSet`. """ @@ -55,7 +55,7 @@ PATCH = "/tmp/scaleswe_f2p.patch" SCORER = "/tmp/scaleswe_scorer.py" -SCORER_SRC = (Path(__file__).parent / "_scorer.py").read_bytes() +SCORER_SRC = (Path(__file__).parent / "score.py").read_bytes() def _image(image_url: str) -> str: diff --git a/examples/tasksets/scaleswe_v1/_scorer.py b/examples/tasksets/scaleswe_v1/score.py similarity index 100% rename from examples/tasksets/scaleswe_v1/_scorer.py rename to examples/tasksets/scaleswe_v1/score.py From 45d621ff706f6f1d09579a44afc86b72fccf6634 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Thu, 11 Jun 2026 00:08:00 +0000 Subject: [PATCH 3/6] feat(v1): NEEDS_CONTAINER taskset flag; trim scaleswe config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add Taskset.NEEDS_CONTAINER ClassVar; the Environment refuses the subprocess runtime for a taskset that sets it. scaleswe-v1 sets NEEDS_CONTAINER = True. - drop scaleswe's dataset_name/split knobs (hardcode AweAI-Team/Scale-SWE train); the taskset uses the base TasksetConfig. - drop the pre_commands guard — all 20181 Scale-SWE rows carry pre_commands. Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/tasksets/scaleswe_v1/scaleswe_v1.py | 14 +++++--------- verifiers/v1/env.py | 8 ++++++++ verifiers/v1/taskset.py | 7 ++++++- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/examples/tasksets/scaleswe_v1/scaleswe_v1.py b/examples/tasksets/scaleswe_v1/scaleswe_v1.py index ac2c2a4cda..8941dfb5aa 100644 --- a/examples/tasksets/scaleswe_v1/scaleswe_v1.py +++ b/examples/tasksets/scaleswe_v1/scaleswe_v1.py @@ -15,6 +15,7 @@ import verifiers.v1 as vf +DATASET = "AweAI-Team/Scale-SWE" REGISTRY = "us-central1-docker.pkg.dev/prime-intellect-platform/prod-sandbox" # The testbed conda env (with the project + pytest installed) and quiet, non-interactive @@ -91,16 +92,13 @@ class ScaleSWETask(vf.Task): pass_to_pass: list[str] = [] -class ScaleSWEConfig(vf.TasksetConfig): - dataset_name: str = "AweAI-Team/Scale-SWE" - split: str = "train" +class ScaleSWETaskset(vf.Taskset[ScaleSWETask, vf.TasksetConfig]): + NEEDS_CONTAINER = True - -class ScaleSWETaskset(vf.Taskset[ScaleSWETask, ScaleSWEConfig]): def load_tasks(self) -> list[ScaleSWETask]: from datasets import load_dataset - rows = load_dataset(self.config.dataset_name, split=self.config.split) + rows = load_dataset(DATASET, split="train") return [ ScaleSWETask( idx=i, @@ -121,8 +119,6 @@ def load_tasks(self) -> list[ScaleSWETask]: ] async def setup(self, task: ScaleSWETask, runtime: vf.Runtime) -> None: - if not task.pre_commands: - raise vf.ProgramError(f"scaleswe row {task.name!r} has no pre_commands") result = await runtime.run(["sh", "-c", task.pre_commands], ENV) if result.exit_code != 0: raise vf.ProgramError( @@ -159,5 +155,5 @@ async def _apply_f2p_patch(self, runtime: vf.Runtime, patch: str) -> None: return -def load_taskset(config: ScaleSWEConfig) -> ScaleSWETaskset: +def load_taskset(config: vf.TasksetConfig) -> ScaleSWETaskset: return ScaleSWETaskset(config) diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 1ad71383d1..1166f65fff 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -142,6 +142,14 @@ def __init__(self, config: EnvConfig) -> None: f"{self.taskset.config.id!r} exposes tool servers (MCP). Run it with a harness " f"that supports task tools (e.g. --harness.id default), or use a taskset without tools." ) + if self.taskset.NEEDS_CONTAINER and isinstance( + self.harness.config.runtime, SubprocessConfig + ): + raise ValueError( + f"Taskset {self.taskset.config.id!r} needs a container runtime " + "(NEEDS_CONTAINER), but the harness runs on the subprocess runtime; " + "use --harness.runtime.type docker or prime." + ) self.harness_timeout = config.timeout.rollout self.scoring_timeout = config.timeout.scoring self.limits = RolloutLimits( diff --git a/verifiers/v1/taskset.py b/verifiers/v1/taskset.py index c95639e043..4514aecec4 100644 --- a/verifiers/v1/taskset.py +++ b/verifiers/v1/taskset.py @@ -17,7 +17,7 @@ import asyncio from collections.abc import Mapping -from typing import Generic, TypeVar +from typing import ClassVar, Generic, TypeVar from pydantic import model_validator from pydantic_config import BaseConfig @@ -90,6 +90,11 @@ class Taskset(Generic[TaskT, ConfigT]): """Generic over its task and config types, so `self.config` and `load_tasks` are fully typed. Subclass: implement `load_tasks`, add @reward/@metric.""" + NEEDS_CONTAINER: ClassVar[bool] = False + """Whether this taskset only runs in a container runtime (docker/prime). When True the + Environment refuses the subprocess runtime — for tasksets whose work only makes sense + inside a per-task image (e.g. a SWE repo sandbox).""" + def __init__(self, config: ConfigT) -> None: self.config = config From 46da6d117d3f2308df2fbb0de5d57e3a46600520 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Thu, 11 Jun 2026 00:13:32 +0000 Subject: [PATCH 4/6] chore: drop the GCP registry prefix from scaleswe images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prime sandbox pulls the raw Docker Hub image (aweaiteam/scaleswe:) directly — verified in a smoke — so the us-central1 prod-sandbox prefix the v0 env prepended is unnecessary. Use the row's image_url as-is. Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/tasksets/scaleswe_v1/scaleswe_v1.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/examples/tasksets/scaleswe_v1/scaleswe_v1.py b/examples/tasksets/scaleswe_v1/scaleswe_v1.py index 8941dfb5aa..61c6ddb8e8 100644 --- a/examples/tasksets/scaleswe_v1/scaleswe_v1.py +++ b/examples/tasksets/scaleswe_v1/scaleswe_v1.py @@ -16,7 +16,6 @@ import verifiers.v1 as vf DATASET = "AweAI-Team/Scale-SWE" -REGISTRY = "us-central1-docker.pkg.dev/prime-intellect-platform/prod-sandbox" # The testbed conda env (with the project + pytest installed) and quiet, non-interactive # tooling — exported for every command the taskset runs in the sandbox. @@ -59,10 +58,6 @@ SCORER_SRC = (Path(__file__).parent / "score.py").read_bytes() -def _image(image_url: str) -> str: - return image_url if image_url.startswith(REGISTRY) else f"{REGISTRY}/{image_url}" - - def _ids(raw: str | list[str] | None) -> list[str]: """F2P/P2P ids arrive as a list or a JSON-encoded string; normalize to list[str].""" if not raw: @@ -104,7 +99,7 @@ def load_tasks(self) -> list[ScaleSWETask]: idx=i, name=row["instance_id"], instruction=row["problem_statement"], - image=_image(row["image_url"]), + image=row["image_url"], workdir=row["workdir"], base_commit=row.get("parent_commit") or row.get("base_commit") or "", pre_commands=(row.get("pre_commands") or "") From 7f3e4bcab553f15b06e00030c60130234bce4426 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Thu, 11 Jun 2026 01:13:47 +0000 Subject: [PATCH 5/6] fix(v1): honor cli/toml workdir over the task's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runtime_for injected task.workdir unconditionally, overriding a user-set --harness.runtime.workdir. Apply the task's workdir only when the runtime config's is still the default — matching the "cli/toml > task > default" precedence the resources loop already uses. Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/env.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 1166f65fff..b462ec6d1f 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -174,7 +174,12 @@ def runtime_for(self, task: Task) -> RuntimeConfig: "runtime has no container; use the docker or prime runtime" ) updates["image"] = task.image - if task.workdir is not None and "workdir" in type(config).model_fields: + workdir_spec = type(config).model_fields.get("workdir") + if ( + task.workdir is not None + and workdir_spec is not None + and getattr(config, "workdir") == workdir_spec.default + ): # cli/toml-set workdir wins over the task's (precedence as for resources) updates["workdir"] = task.workdir for field, value in task.resources.model_dump(exclude_none=True).items(): spec = type(config).model_fields.get(field) From 681877932c9fe258e9c174c069f1840de654df44 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Thu, 11 Jun 2026 01:13:58 +0000 Subject: [PATCH 6/6] chore: drop redundant comment on workdir precedence Co-Authored-By: Claude Opus 4.8 (1M context) --- verifiers/v1/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index b462ec6d1f..5e29946cba 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -179,7 +179,7 @@ def runtime_for(self, task: Task) -> RuntimeConfig: task.workdir is not None and workdir_spec is not None and getattr(config, "workdir") == workdir_spec.default - ): # cli/toml-set workdir wins over the task's (precedence as for resources) + ): updates["workdir"] = task.workdir for field, value in task.resources.model_dump(exclude_none=True).items(): spec = type(config).model_fields.get(field)