diff --git a/docs/source/environments/tbench2.md b/docs/source/environments/tbench2.md index f4cd559fa..ee2132e5e 100644 --- a/docs/source/environments/tbench2.md +++ b/docs/source/environments/tbench2.md @@ -96,7 +96,7 @@ TB2_MODE=local python -m tbench2_env.server.app ### Docker Mode -Each task runs in its own Docker container, using the image specified in the task's `task.toml`: +Each task runs in its own Docker container, using the image specified in the task's `task.toml` (tasks that declare no `docker_image` are rejected at reset — there is no host-execution fallback). Agent commands run in the image's own `WORKDIR`, and `evaluate` scores with the task's canonical `tests/test.sh`, staged for exactly the verify window — the same scoring contract as local mode: ```bash # Enable Docker mode diff --git a/envs/tbench2_env/README.md b/envs/tbench2_env/README.md index b36266883..d2d4a2a38 100644 --- a/envs/tbench2_env/README.md +++ b/envs/tbench2_env/README.md @@ -110,7 +110,7 @@ TB2_MODE=local python -m tbench2_env.server.app ### Docker Mode -Each task runs in its own Docker container, using the image specified in the task's `task.toml`: +Each task runs in its own Docker container, using the image specified in the task's `task.toml` (tasks that declare no `docker_image` are rejected at reset — there is no host-execution fallback). Agent commands run in the image's own `WORKDIR`, and `evaluate` scores with the task's canonical `tests/test.sh`, staged for exactly the verify window — the same scoring contract as local mode: ```bash # Enable Docker mode diff --git a/envs/tbench2_env/server/app.py b/envs/tbench2_env/server/app.py index 67486680b..b4e8fb7f0 100644 --- a/envs/tbench2_env/server/app.py +++ b/envs/tbench2_env/server/app.py @@ -28,7 +28,7 @@ python -m server.app Environment Variables: - TB2_MODE: Execution mode - "local" (default), "docker", or "auto" + TB2_MODE: Execution mode - "local" (default) or "docker" MAX_CONCURRENT_ENVS: Maximum concurrent WebSocket sessions (default: 8) """ @@ -61,13 +61,15 @@ if _TB2_MODE == "docker": _DEFAULT_ENVIRONMENT = Tbench2DockerEnvironment _ENV_SUFFIX = " (Docker mode)" -elif _TB2_MODE == "auto": - # Auto-detect: try Docker, fall back to local - _DEFAULT_ENVIRONMENT = Tbench2Environment - _ENV_SUFFIX = " (auto-detect mode)" -else: +elif _TB2_MODE == "local": _DEFAULT_ENVIRONMENT = Tbench2Environment _ENV_SUFFIX = " (local mode)" +else: + # No silent aliases: the old "auto" value claimed to auto-detect Docker + # but always ran local mode, and a typo'd mode would quietly do the same. + # Which class serves decides the isolation and scoring contract, so + # refuse to guess. + raise ValueError(f"Unknown TB2_MODE {_TB2_MODE!r}: expected 'local' or 'docker'.") # Create the app with web interface and README integration diff --git a/envs/tbench2_env/server/tbench2_env_environment.py b/envs/tbench2_env/server/tbench2_env_environment.py index 44a0fd0cc..a232b95c9 100644 --- a/envs/tbench2_env/server/tbench2_env_environment.py +++ b/envs/tbench2_env/server/tbench2_env_environment.py @@ -165,6 +165,79 @@ def _read_timeout(task_dir: Path, fallback: float) -> float: return float(verifier.get("timeout_sec", fallback)) +# The scoring exec echoes its verdict on a marker line so the caller can parse +# it out of mixed stdout. Shared by both execution modes (local terminal +# toolkit and Docker container exec). +_REWARD_MARKER = "__TB2_REWARD__:" +_EXIT_CODE_MARKER = "__TB2_EXIT_CODE__:" + + +def _canonical_eval_cmd(workdir: str, timeout_s: float | None = None) -> str: + """The official-harness scoring command, run from the agent's workdir. + + /tests/test.sh (staged by the caller) pins its own pytest toolchain (uvx: + Python 3.13 + pytest 8.4.1 + ctrf), runs tests/test_outputs.py from + _VERIFY_TESTS_DIR against the task workdir, and writes the binary result + to /logs/verifier/reward.txt. test.sh stdout goes under _VERIFIER_LOG_DIR + (wiped with the verify window) rather than a fixed /tmp path that would + outlive scoring: its pytest -rA output can spell out expected values. Its + tail is echoed back into the returned output — the episode is over by + then, and callers need the pytest diagnostics on failure; the reward + marker line stays last for parsing. + + ``timeout_s`` bounds test.sh with coreutils ``timeout`` when present in + the image. The local mode passes None: its terminal toolkit enforces the + budget itself. Docker exec has no server-side timeout, so the task's own + verifier budget is enforced in-shell. + """ + run = f"bash {_VERIFY_TESTS_DIR}/test.sh" + if timeout_s is not None: + run = f"if command -v timeout >/dev/null 2>&1; then timeout {int(timeout_s)} {run}; else {run}; fi" + return ( + f"cd {shlex.quote(workdir)} && " + f"{run} > {_VERIFIER_LOG_DIR}/testsh.log 2>&1; " + f"tail -c 20000 {_VERIFIER_LOG_DIR}/testsh.log 2>/dev/null; " + f"echo {_REWARD_MARKER}$(cat {_VERIFIER_LOG_DIR}/reward.txt 2>/dev/null)" + ) + + +def _parse_canonical_reward(output: str) -> float: + for line in output.splitlines()[::-1]: + if _REWARD_MARKER in line: + raw = line.split(_REWARD_MARKER, 1)[1].strip() + try: + return float(raw) if raw else 0.0 + except ValueError: + return 0.0 + return 0.0 + + +def _fallback_eval_cmd(workdir: str) -> str: + """pytest against the staged tests copy, for task dirs without the + canonical harness (none of the 89 official TB2 tasks — they all ship + test.sh — but custom task dirs may only have bare pytest tests). Prefer + uvx so pytest comes with its own toolchain like the canonical harness + does. Verify from the same directory the agent worked in. + """ + return ( + f"cd {shlex.quote(workdir)} && " + "if command -v uvx >/dev/null 2>&1; " + f"then uvx --with pytest==8.4.1 pytest -q {_VERIFY_TESTS_DIR} -rA; " + f"else python -m pytest -q {_VERIFY_TESTS_DIR} -rA; fi; " + f"echo {_EXIT_CODE_MARKER}$?" + ) + + +def _parse_exit_code_marker(output: str) -> int: + for line in output.splitlines()[::-1]: + if _EXIT_CODE_MARKER in line: + try: + return int(line.split(_EXIT_CODE_MARKER, 1)[1].strip()) + except Exception: + return 1 + return 1 + + class Tbench2Environment(Environment[Tbench2Action, Tbench2Observation, Tbench2State]): """OpenEnv wrapper around Terminal-Bench 2 tasks (local execution).""" @@ -541,72 +614,32 @@ def _evaluate_canonical( whenever the task ships it. """ # Same working dir the agent operated in (resolved during reset). + # The toolkit enforces the verifier budget itself, so the command + # carries no in-shell timeout (see _canonical_eval_cmd). workdir = self._workdir or str(self._task_dir) - marker = "__TB2_REWARD__:" - # test.sh stdout goes under /logs/verifier (wiped with the verify - # window in _evaluate_task) rather than a fixed /tmp path that would - # outlive scoring: its pytest -rA output can spell out expected values. - # Its tail is read back into the returned output — the episode is over - # by then, and callers need the pytest diagnostics on failure; the - # reward marker line stays last for parsing. - cmd = ( - f"cd {shlex.quote(workdir)} && " - f"bash {_VERIFY_TESTS_DIR}/test.sh > {_VERIFIER_LOG_DIR}/testsh.log 2>&1; " - f"tail -c 20000 {_VERIFIER_LOG_DIR}/testsh.log 2>/dev/null; " - f"echo {marker}$(cat {_VERIFIER_LOG_DIR}/reward.txt 2>/dev/null)" - ) output = self._terminal_toolkit.shell_exec( id="tb2-tests", - command=cmd, + command=_canonical_eval_cmd(workdir), block=True, timeout=timeout_s, ) - reward = 0.0 - for line in output.splitlines()[::-1]: - if marker in line: - raw = line.split(marker, 1)[1].strip() - try: - reward = float(raw) if raw else 0.0 - except ValueError: - reward = 0.0 - break - + reward = _parse_canonical_reward(output) info = {"tests_passed": reward == 1.0, "harness": "tests/test.sh"} return output, reward, info def _evaluate_fallback(self, timeout_s: float) -> tuple[str, float, dict[str, Any]]: - """For task dirs without the canonical harness (none of the 89 official - TB2 tasks — they all ship test.sh — but custom task dirs may only have - bare pytest tests). Runs pytest against the staged /tests copy; prefer - uvx so pytest comes with its own toolchain like the canonical harness - does. Verify from the same directory the agent worked in. - """ + """Scoring for task dirs without the canonical harness (see + _fallback_eval_cmd).""" fallback_cwd = self._workdir or str(self._task_dir) - cmd = ( - f"cd {shlex.quote(fallback_cwd)} && " - "if command -v uvx >/dev/null 2>&1; " - f"then uvx --with pytest==8.4.1 pytest -q {_VERIFY_TESTS_DIR} -rA; " - f"else python -m pytest -q {_VERIFY_TESTS_DIR} -rA; fi; " - "echo __TB2_EXIT_CODE__:$?" - ) output = self._terminal_toolkit.shell_exec( id="tb2-tests", - command=cmd, + command=_fallback_eval_cmd(fallback_cwd), block=True, timeout=timeout_s, ) - exit_code = 1 - marker = "__TB2_EXIT_CODE__" - for line in output.splitlines()[::-1]: - if marker in line: - try: - exit_code = int(line.split(":", 1)[1].strip()) - except Exception: - exit_code = 1 - break - + exit_code = _parse_exit_code_marker(output) reward = 1.0 if exit_code == 0 else 0.0 info = {"tests_passed": exit_code == 0, "exit_code": exit_code} return output, reward, info @@ -617,8 +650,14 @@ class Tbench2DockerEnvironment( ): """OpenEnv wrapper around Terminal-Bench 2 tasks with Docker isolation. - This environment runs each task in its own Docker container, reading - the image specification from task.toml's [environment] section. + This environment runs each task in its own Docker container built from + the task's official image (task.toml's [environment] docker_image — tasks + without one are rejected at reset; there is no host-execution fallback). + Agent commands run in the image's own WORKDIR, and ``evaluate`` scores + with the task's canonical tests/test.sh staged at the official fixed + paths for exactly the verify window — the same scoring contract as the + local mode, just delivered over container exec instead of the in-process + terminal toolkit. Requires: - Docker socket mounted (/var/run/docker.sock) @@ -653,6 +692,7 @@ def __init__( self._instruction = "" self._task_image = "" self._task_config: dict[str, Any] = {} + self._workdir = "" def _get_docker_client(self) -> Any: """Lazy initialization of Docker client.""" @@ -686,36 +726,48 @@ def reset( # Read task configuration including Docker image task_toml_path = task_dir / "task.toml" if task_toml_path.exists(): - self._task_config = tomllib.loads( - task_toml_path.read_text(encoding="utf-8") - ) - self._task_image = self._task_config.get("environment", {}).get( - "docker_image", "" - ) + task_config = tomllib.loads(task_toml_path.read_text(encoding="utf-8")) + task_image = task_config.get("environment", {}).get("docker_image", "") else: - self._task_image = "" - self._task_config = {} - - self._instruction = _read_instruction(task_dir) - self._task_dir = task_dir + task_image = "" + task_config = {} + + instruction = _read_instruction(task_dir) + + # No host-execution fallback: silently running agent commands on the + # env-server host would be a containment hole (arbitrary agent shell + # outside any container) and a scoring-fidelity hole (no official + # image, no canonical harness) at once. Every official TB2 task + # declares its image; a task dir that doesn't is a bug to surface. + if not task_image: + # Fail closed: a rejected reset must not leave a previous + # container usable with metadata from this new task. + self.close() + raise RuntimeError( + f"task {resolved_task_id} declares no [environment] docker_image " + "in task.toml; Docker mode runs every episode in the task's " + "official container and does not fall back to executing on the " + "server host. Fix the task, or use TB2_MODE=local." + ) # Create trial directory for logs trial_name = f"{resolved_task_id}.{episode_id or uuid4().hex}" trial_dir = self.output_dir / trial_name trial_dir.mkdir(parents=True, exist_ok=True) - # Start Docker container if image is specified - if self._task_image: + # A successful reset replaces the previous episode. Commit the new + # metadata only after all task inputs have been validated, and leave + # the session closed if container startup fails. + self.close() + self._task_config = task_config + self._task_image = task_image + self._instruction = instruction + self._task_dir = task_dir + try: self._start_container(task_dir, trial_dir) - else: - # Fallback to local mode if no image specified - self._state = Tbench2State( - episode_id=episode_id or str(uuid4()), - step_count=0, - task_id=resolved_task_id, - task_path=str(task_dir), - terminal_ready=not self._task_image, # Ready if no container needed - ) + except Exception: + self.close() + raise return Tbench2Observation( instruction=self._instruction, @@ -726,7 +778,7 @@ def reset( task_path=str(task_dir), session_id=None, action_type="reset", - info={"docker_image": self._task_image} if self._task_image else {}, + info={"docker_image": self._task_image}, reward=0.0, done=False, ) @@ -743,12 +795,18 @@ def _start_container(self, task_dir: Path, trial_dir: Path) -> None: try: # Pull image if needed try: - docker.images.get(self._task_image) + image = docker.images.get(self._task_image) except Exception: logging.info(f"Pulling image {self._task_image}...") - docker.images.pull(self._task_image) + image = docker.images.pull(self._task_image) + if isinstance(image, list): # pull without a tag returns a list + image = image[0] - # Start container WITHOUT bind mounts (for DinD compatibility) + self._workdir = self._resolve_workdir(image, task_dir) + + # Start container WITHOUT bind mounts (for DinD compatibility). + # working_dir=/task only guarantees the task-source copy target + # exists; every exec cd's into the resolved image WORKDIR. self._container = docker.containers.run( image=self._task_image, command="sleep infinity", @@ -808,16 +866,37 @@ def _copy_dir_to_container( # Copy to container self._container.put_archive(dest_path, tar_stream.getvalue()) + def _resolve_workdir(self, image: Any, task_dir: Path) -> str: + """The dir agent commands and scoring run in: the task image's own + WORKDIR — the real TB2 task state, not the /task source copy. + + Image metadata is authoritative (it sees a WORKDIR inherited from a + base image, which Dockerfile parsing misses); fall back to parsing + the task's Dockerfile, then to /task. The local mode's server-tree + guard does not apply here: this server sits outside the container, + so the image's /app is always the task tree. + """ + try: + workdir = (image.attrs.get("Config") or {}).get("WorkingDir") or "" + except Exception: + workdir = "" + return workdir or _task_image_workdir(task_dir) or "/task" + def _exec_in_container( - self, command: str, workdir: str = "/task" + self, command: str, workdir: str | None = None ) -> tuple[int, str]: - """Execute a command inside the container.""" + """Execute a command inside the container, from the task workdir. + + The command rides as an argv element (bash -c ), not spliced + into a quoted shell string — an agent command containing a single + quote must arrive in the container byte-identical. + """ if self._container is None: raise RuntimeError("Container not started. Call reset() first.") + cwd = workdir or self._workdir or "/task" exit_code, output = self._container.exec_run( - cmd=f"bash -c 'cd {workdir} && {command}'", - workdir="/task", + cmd=["bash", "-c", f"cd {shlex.quote(cwd)} && {command}"], stdout=True, stderr=True, ) @@ -851,41 +930,18 @@ def step( try: if action.action_type == "exec": - if self._container: - exit_code, output = self._exec_in_container(action.command) - success = exit_code == 0 - else: - # Fallback to local execution - import subprocess - - result = subprocess.run( - action.command, - shell=True, - capture_output=True, - text=True, - timeout=self.command_timeout_s, - ) - output = result.stdout + result.stderr - success = result.returncode == 0 + exit_code, output = self._exec_in_container(action.command) + success = exit_code == 0 elif action.action_type == "write_file": - if self._container: - # Write to container - exit_code, _ = self._exec_in_container( - f"cat > {action.file_path} << 'EOF'\n{action.content}\nEOF" - ) - success = exit_code == 0 - output = f"Wrote to {action.file_path}" - else: - # Local write - Path(action.file_path).write_text(action.content) - output = f"Wrote to {action.file_path}" + exit_code, _ = self._exec_in_container( + f"cat > {action.file_path} << 'EOF'\n{action.content}\nEOF" + ) + success = exit_code == 0 + output = f"Wrote to {action.file_path}" elif action.action_type == "evaluate": - if self._container: - output, reward, info = self._evaluate_docker() - else: - output, reward, info = self._evaluate_local() + output, reward, info = self._evaluate_docker() done = True elif action.action_type == "close": @@ -920,7 +976,17 @@ def step( ) def _evaluate_docker(self) -> tuple[str, float, dict[str, Any]]: - """Evaluate task inside Docker container.""" + """Score with the canonical harness inside the task container. + + Same scoring contract as the local mode's _evaluate_task: stage a + pristine tests/ copy at the official fixed path for exactly the + verify window, run the task's tests/test.sh from the dir the agent + worked in (the image WORKDIR), read the binary verdict from + /logs/verifier/reward.txt; task dirs without test.sh fall back to + pytest. No cross-session lock is needed here, unlike local mode: + every session drives its OWN container, so the fixed paths are + session-private. + """ if self._container is None: raise RuntimeError("Container not started.") assert self._task_dir is not None, "Task directory not set" @@ -928,8 +994,10 @@ def _evaluate_docker(self) -> tuple[str, float, dict[str, Any]]: # Stage-at-verify, like the official TB2 harness: the initial /task # copy excludes tests/, and this server sits OUTSIDE the container, so # the copy staged here is one the agent never saw. rm -rf first so - # nothing an agent pre-planted at /task/tests (e.g. a conftest.py - # pytest would auto-load) survives into scoring. + # nothing an agent pre-planted at the fixed paths survives into + # scoring — a /tests/conftest.py pytest would auto-load, a symlink + # redirecting the stage at a dir the agent controls, or a stale + # /logs/verifier/reward.txt read back as a verdict. tests_src = self._task_dir / "tests" if not tests_src.is_dir(): return ( @@ -937,86 +1005,60 @@ def _evaluate_docker(self) -> tuple[str, float, dict[str, Any]]: 0.0, {"tests_passed": False, "error": "missing tests"}, ) + + # The task's own verifier budget (task.toml [verifier].timeout_sec) — + # heavy tests legitimately run minutes (circuit-fibsqrt declares 3600s). + verifier_timeout_s = _read_timeout(self._task_dir, fallback=900.0) + workdir = self._workdir or "/task" + wipe_ec, wipe_out = self._exec_in_container( - "rm -rf /task/tests && mkdir -p /task/tests" + f"rm -rf {_VERIFY_TESTS_DIR} {_VERIFIER_LOG_DIR} && " + f"mkdir -p {_VERIFY_TESTS_DIR} {_VERIFIER_LOG_DIR}" ) if wipe_ec != 0: # Fail closed: put_archive into a dir that survived the wipe would # merge the staged tests into whatever the agent planted there. raise RuntimeError( - f"could not reset /task/tests before verify: {wipe_out.strip()}" + f"could not reset {_VERIFY_TESTS_DIR} before verify: {wipe_out.strip()}" ) try: - self._copy_dir_to_container(tests_src, "/task/tests") + self._copy_dir_to_container(tests_src, _VERIFY_TESTS_DIR) - # Run pytest in the container's /task directory - # Use exit code marker for consistency with local mode - cmd = ( - "cd /task && python -m pytest -q tests/ -rA; echo __TB2_EXIT_CODE__:$?" - ) - - exit_code, output = self._container.exec_run( - cmd=f"bash -c '{cmd}'", - workdir="/task", - stdout=True, - stderr=True, - ) - output_str = output.decode("utf-8", errors="replace") - - # Parse exit code from marker (same logic as local mode) - ec = 1 - marker = "__TB2_EXIT_CODE__" - for line in output_str.splitlines()[::-1]: - if marker in line: - try: - ec = int(line.split(":", 1)[1].strip()) - except Exception: - ec = 1 - break + if (tests_src / "test.sh").is_file(): + _, output = self._exec_in_container( + _canonical_eval_cmd(workdir, timeout_s=verifier_timeout_s) + ) + reward = _parse_canonical_reward(output) + info = {"tests_passed": reward == 1.0, "harness": "tests/test.sh"} + else: + _, output = self._exec_in_container(_fallback_eval_cmd(workdir)) + exit_code = _parse_exit_code_marker(output) + reward = 1.0 if exit_code == 0 else 0.0 + info = {"tests_passed": exit_code == 0, "exit_code": exit_code} finally: - # Tests live in the container only for the verify window, matching - # the local-mode staging: don't leave /task/tests readable - # afterward — the agent keeps its session if scoring errored - # (step() reports the failure without ending the episode). + # Verifier artifacts live in the container only for the verify + # window, matching the local-mode staging: neither the staged + # tests (expected values) nor /logs/verifier (reward + pytest -rA + # log) may stay readable afterward — the agent keeps its session + # if scoring errored (step() reports the failure without ending + # the episode). try: - rm_ec, rm_out = self._exec_in_container("rm -rf /task/tests") + rm_ec, rm_out = self._exec_in_container( + f"rm -rf {_VERIFY_TESTS_DIR} {_VERIFIER_LOG_DIR}" + ) if rm_ec != 0: logging.warning( - "failed to remove staged /task/tests after verify: %s", + "failed to remove staged %s after verify: %s", + _VERIFY_TESTS_DIR, rm_out.strip(), ) except Exception: logging.warning( - "failed to remove staged /task/tests after verify", + "failed to remove staged %s after verify", + _VERIFY_TESTS_DIR, exc_info=True, ) - reward = 1.0 if ec == 0 else 0.0 - info = {"tests_passed": ec == 0, "exit_code": ec} - return output_str, reward, info - - def _evaluate_local(self) -> tuple[str, float, dict[str, Any]]: - """Evaluate task locally (fallback).""" - if self._task_dir is None: - raise RuntimeError("Task not initialized.") - - tests_dir = self._task_dir / "tests" - cmd = f"cd {self._task_dir} && python -m pytest -q {tests_dir} -rA; echo __TB2_EXIT_CODE__:$?" - - import subprocess - - result = subprocess.run( - cmd, - shell=True, - capture_output=True, - text=True, - timeout=900.0, - ) - output = result.stdout + result.stderr - exit_code = result.returncode - - reward = 1.0 if exit_code == 0 else 0.0 - info = {"tests_passed": exit_code == 0, "exit_code": exit_code} return output, reward, info @property @@ -1033,6 +1075,7 @@ def close(self) -> None: self._container = None self._task_dir = None self._instruction = "" + self._workdir = "" def _resolve_task_path(self, task_id: str | None, task_path: str | None) -> Path: if task_path: diff --git a/tests/envs/test_tbench2_env.py b/tests/envs/test_tbench2_env.py index f0afdb064..9640821e8 100644 --- a/tests/envs/test_tbench2_env.py +++ b/tests/envs/test_tbench2_env.py @@ -336,10 +336,15 @@ def test_withhold_removes_symlinked_tests(tmp_path: Path): class _FakeContainer: - """Records exec/put_archive calls in one ordered event log.""" + """Records exec/put_archive calls in one ordered event log. - def __init__(self, exec_output: bytes = b"__TB2_EXIT_CODE__:0\n"): + exec_run receives argv form (["bash", "-c", payload]); the log records + the shell payload, and raw_cmds keeps the argv for shape assertions. + """ + + def __init__(self, exec_output: bytes = b"__TB2_REWARD__:1\n"): self.events: list[tuple] = [] + self.raw_cmds: list = [] self.exec_output = exec_output def put_archive(self, dest, data): @@ -347,7 +352,8 @@ def put_archive(self, dest, data): return True def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): - self.events.append(("exec", cmd)) + self.raw_cmds.append(cmd) + self.events.append(("exec", cmd[-1] if isinstance(cmd, list) else cmd)) return 0, self.exec_output @@ -385,15 +391,55 @@ def test_evaluate_docker_stages_tests_at_verify(tmp_path: Path): output, reward, info = env._evaluate_docker() assert reward == 1.0 + assert info == {"tests_passed": True, "harness": "tests/test.sh"} kinds = [e[0] for e in container.events] - # rm -rf the agent-writable staging dir BEFORE the fresh copy goes in, - # the copy lands before pytest runs, and tests are removed again after. + # rm -rf the agent-writable fixed paths BEFORE the fresh copy goes in, + # the copy lands before test.sh runs, and both are removed again after. assert kinds == ["exec", "put", "exec", "exec"] - assert "rm -rf /task/tests" in container.events[0][1] - assert container.events[1][1] == "/task/tests" + assert "rm -rf /tests /logs/verifier" in container.events[0][1] + assert container.events[1][1] == "/tests" assert "test.sh" in _tar_names(container.events[1][2]) - assert "pytest" in container.events[2][1] - assert "rm -rf /task/tests" in container.events[3][1] + eval_cmd = container.events[2][1] + # Canonical harness from the agent's workdir, bounded by the verifier + # budget, verdict read from reward.txt — not bare pytest in /task. + assert "bash /tests/test.sh" in eval_cmd + assert eval_cmd.startswith("cd /task && ") # no resolved workdir → /task + assert "timeout 900" in eval_cmd + assert "/logs/verifier/reward.txt" in eval_cmd + assert "rm -rf /tests /logs/verifier" in container.events[3][1] + + +def test_evaluate_docker_runs_in_resolved_workdir(tmp_path: Path): + task = _make_task_dir(tmp_path) + env = Tbench2DockerEnvironment() + env._container = _FakeContainer() + env._task_dir = task + env._workdir = "/app" + + _, reward, _ = env._evaluate_docker() + + assert reward == 1.0 + eval_cmd = env._container.events[2][1] + assert eval_cmd.startswith("cd /app && ") + + +def test_evaluate_docker_fallback_without_testsh(tmp_path: Path): + """A task dir whose tests/ ships no canonical harness scores via pytest, + still against the staged /tests copy.""" + task = _make_task_dir(tmp_path) + (task / "tests" / "test.sh").unlink() + env = Tbench2DockerEnvironment() + container = _FakeContainer(exec_output=b"__TB2_EXIT_CODE__:0\n") + env._container = container + env._task_dir = task + + output, reward, info = env._evaluate_docker() + + assert reward == 1.0 + assert info == {"tests_passed": True, "exit_code": 0} + eval_cmd = container.events[2][1] + assert "pytest -q /tests -rA" in eval_cmd + assert "test.sh" not in eval_cmd def test_evaluate_docker_cleans_up_when_scoring_raises(tmp_path: Path): @@ -404,8 +450,9 @@ def test_evaluate_docker_cleans_up_when_scoring_raises(tmp_path: Path): class _ExplodingContainer(_FakeContainer): def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): - if "pytest" in cmd: - self.events.append(("exec", cmd)) + payload = cmd[-1] if isinstance(cmd, list) else cmd + if "test.sh" in payload: + self.events.append(("exec", payload)) raise RuntimeError("docker daemon hiccup") return super().exec_run(cmd, workdir=workdir, stdout=stdout, stderr=stderr) @@ -418,7 +465,7 @@ def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): env._evaluate_docker() assert container.events[-1][0] == "exec" - assert "rm -rf /task/tests" in container.events[-1][1] + assert "rm -rf /tests /logs/verifier" in container.events[-1][1] def test_evaluate_docker_fails_closed_when_prestage_wipe_fails(tmp_path: Path): @@ -428,9 +475,10 @@ def test_evaluate_docker_fails_closed_when_prestage_wipe_fails(tmp_path: Path): class _StubbornContainer(_FakeContainer): def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): - self.events.append(("exec", cmd)) - if "rm -rf /task/tests" in cmd and "mkdir" in cmd: - return 1, b"rm: cannot remove '/task/tests': busy\n" + payload = cmd[-1] if isinstance(cmd, list) else cmd + self.events.append(("exec", payload)) + if "rm -rf /tests" in payload and "mkdir" in payload: + return 1, b"rm: cannot remove '/tests': busy\n" return 0, self.exec_output env = Tbench2DockerEnvironment() @@ -438,7 +486,7 @@ def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): env._container = container env._task_dir = task - with pytest.raises(RuntimeError, match="could not reset /task/tests"): + with pytest.raises(RuntimeError, match="could not reset /tests"): env._evaluate_docker() assert not any(e[0] == "put" for e in container.events) @@ -450,9 +498,10 @@ def test_evaluate_docker_warns_when_cleanup_fails(tmp_path: Path, caplog): class _LeakyContainer(_FakeContainer): def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): - self.events.append(("exec", cmd)) - if "rm -rf /task/tests" in cmd and "mkdir" not in cmd: - return 1, b"rm: cannot remove '/task/tests': busy\n" + payload = cmd[-1] if isinstance(cmd, list) else cmd + self.events.append(("exec", payload)) + if "rm -rf /tests" in payload and "mkdir" not in payload: + return 1, b"rm: cannot remove '/tests': busy\n" return 0, self.exec_output env = Tbench2DockerEnvironment() @@ -465,7 +514,7 @@ def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): assert reward == 1.0 assert any( - "failed to remove staged /task/tests" in record.message + "failed to remove staged /tests" in record.getMessage() for record in caplog.records ) @@ -484,6 +533,98 @@ def test_evaluate_docker_missing_tests_scores_zero(tmp_path: Path): assert container.events == [] +def test_docker_reset_rejects_task_without_image(tmp_path: Path): + """No host-execution fallback: a task dir that declares no docker_image + must fail reset loudly, not silently run agent commands on the server.""" + task = tmp_path / "imageless-task" + task.mkdir() + (task / "task.toml").write_text("[metadata]\n") + (task / "instruction.md").write_text("do it\n") + + env = Tbench2DockerEnvironment( + tasks_dir=str(tmp_path), output_dir=str(tmp_path / "runs") + ) + + with pytest.raises(RuntimeError, match="docker_image"): + env.reset(task_id="imageless-task") + + +def test_docker_failed_reset_closes_previous_container(tmp_path: Path): + """A rejected reset must not leave the previous container usable with + metadata from the new task.""" + task = tmp_path / "imageless-task" + task.mkdir() + (task / "task.toml").write_text("[metadata]\n") + (task / "instruction.md").write_text("do it\n") + + class _PreviousContainer: + def __init__(self): + self.events = [] + + def stop(self, timeout): + self.events.append(("stop", timeout)) + + def remove(self, force): + self.events.append(("remove", force)) + + env = Tbench2DockerEnvironment( + tasks_dir=str(tmp_path), output_dir=str(tmp_path / "runs") + ) + previous_container = _PreviousContainer() + env._container = previous_container + env._task_dir = tmp_path / "previous-task" + env._instruction = "previous task" + env._workdir = "/previous-workdir" + + with pytest.raises(RuntimeError, match="docker_image"): + env.reset(task_id="imageless-task") + + assert previous_container.events == [("stop", 10), ("remove", True)] + assert env._container is None + assert env._task_dir is None + assert env._instruction == "" + assert env._workdir == "" + + +class _FakeImage: + def __init__(self, working_dir: str): + self.attrs = {"Config": {"WorkingDir": working_dir}} + + +def test_docker_workdir_prefers_image_metadata(tmp_path: Path): + """Image Config.WorkingDir wins (it sees base-image WORKDIRs); the task + Dockerfile is the fallback, /task the last resort.""" + task = _make_task_dir(tmp_path) + (task / "environment").mkdir() + (task / "environment" / "Dockerfile").write_text( + "FROM debian:12\nWORKDIR /from-dockerfile\n" + ) + env = Tbench2DockerEnvironment() + + assert env._resolve_workdir(_FakeImage("/from-image"), task) == "/from-image" + assert env._resolve_workdir(_FakeImage(""), task) == "/from-dockerfile" + + bare = tmp_path / "bare-task" + bare.mkdir() + assert env._resolve_workdir(_FakeImage(""), bare) == "/task" + + +def test_docker_exec_passes_command_as_argv(tmp_path: Path): + """Agent commands ride as a bash -c argv element, so shell quoting in the + command (a single quote, say) survives byte-identical; the exec cd's into + the resolved image workdir.""" + env = Tbench2DockerEnvironment() + container = _FakeContainer() + env._container = container + env._workdir = "/app" + + env._exec_in_container("echo 'hi there'") + + (raw,) = container.raw_cmds + assert isinstance(raw, list) and raw[:2] == ["bash", "-c"] + assert raw[2] == "cd /app && echo 'hi there'" + + @pytest.mark.skipif(camel is None, reason="camel-ai not installed") @pytest.mark.skipif( os.environ.get("TB2_ENABLE_TESTS", "0") != "1",