openenv/tbench2: score the shared-server leg natively; retire the adapter compensation

examples/experimental/openenv/README.md

@@ -50,6 +50,10 @@ concurrency. Per-task containers are heavy on disk — if you'd rather not coloc
 them with the GPU workload, run the env server on a separate Docker host and point
 the launcher at it via `--openenv-env-url http://<env-host>:8003`.
 
+The installed `tbench2_env` must be `>=` the #1012 merge (04d259ea6), same as
+step 2b below; the adapter drops every episode (with a warning) from a server
+that doesn't carry that contract.
+
 ### 2b. Alternative: Daytona cloud sandboxes (no Docker host)
 
 Instead of one shared env server, the adapter can give **every episode its own
@@ -74,7 +78,7 @@ from upstream main (editable: the recipe embeds the package source, which
 needs `pyproject.toml` present next to the package):
 
 ```bash
-git clone https://github.com/huggingface/OpenEnv.git   # >= the #965/#972 merge (39c91bfd); pin that sha if you need frozen reward semantics across a long run
+git clone https://github.com/huggingface/OpenEnv.git   # >= the #1012 merge (04d259ea6, the full canonical contract for both modes); pin that sha if you need frozen reward semantics across a long run
 pip install -e OpenEnv/envs/tbench2_env
 ```
 

examples/experimental/openenv/openenv_daytona_agent_function.py

@@ -10,14 +10,13 @@
 
 The agent loop and training wrapper live in ``openenv_agent_function``
 (sibling module) and are reused unchanged; this module only supplies its own
-``run_episode`` — how an env comes into being and which server contract it
-speaks (``native_evaluate``, see the episode-wiring note there). The image
-recipe lives in ``tb2_sandbox_recipe`` and its Daytona materialization in
-``tb2_sandbox_daytona``; the recipe bakes the installed ``tbench2_env``
-package -- OpenEnv's Terminal-Bench-2 environment package -- into the image,
-so this variant needs the pinned tbench2_env install from the README
-(canonical test.sh scoring and verifier-asset withholding built into the
-server).
+``run_episode`` — how an env comes into being (see the episode-wiring note
+there). The image recipe lives in ``tb2_sandbox_recipe`` and its Daytona
+materialization in ``tb2_sandbox_daytona``; the recipe bakes the installed
+``tbench2_env`` package -- OpenEnv's Terminal-Bench-2 environment package --
+into the image, so this variant needs the pinned tbench2_env install from
+the README (canonical test.sh scoring and verifier-asset withholding built
+into the server).
 
 Env vars (the agent-loop ones in ``openenv_agent_function`` apply too):
   OPENENV_TB2_TASKS_DIR        path to a terminal-bench-2 checkout: build the
@@ -59,13 +58,12 @@
 # definition, read off the local TB2 checkout (OPENENV_TB2_TASKS_DIR); repeat
 # creates hit Daytona's build cache, and no named snapshot is involved.
 #
-# The sandbox's env server is the CURRENT upstream tbench2_env baked by the
-# recipe — carrying the fixes upstreamed via huggingface/OpenEnv#965 + #972:
-# canonical tests/test.sh scoring built into `evaluate`, task WORKDIR resolved
-# server-side, verifier assets withheld. So run_episode here sets
-# native_evaluate=True and the adapter-side compensation machinery in
-# openenv_agent_function (_apply_workdir / _CANONICAL_EVAL_CMD) is deliberately
-# not applied — the launcher preflight rejects an older install outright.
+# The sandbox's env server is the tbench2_env baked by the recipe, installed
+# per the README (at or after the huggingface/OpenEnv#1012 merge): canonical
+# tests/test.sh scoring built into `evaluate`, task WORKDIR resolved
+# server-side, verifier assets withheld. The launcher preflight rejects an
+# older install outright, and the shared agent loop's harness-marker guard
+# backstops it per episode.
 #
 # Daytona rate-limits sandbox creation (ThrottlerException: Too Many Requests).
 # A rollout fans out many episodes at once; cap in-flight creates process-wide
@@ -219,10 +217,7 @@ async def run_episode(
     """One episode in its own Daytona sandbox, with the caller's own
     policy. Direct-drive entry, same contract as openenv_agent_function's.
 
-    native_evaluate=True: the baked server carries the OpenEnv#965/#972 fixes —
-    raw exec commands (WORKDIR resolved server-side), scoring via the native
-    `evaluate` action. No post-episode hygiene: the sandbox is deleted when
-    the episode ends.
+    No post-episode hygiene: the sandbox is deleted when the episode ends.
     """
     return await oaf._multi_turn(
         oaf._load_tbench2(),
@@ -232,7 +227,6 @@ async def run_episode(
         request_kwargs,
         metadata,
         run_body=_sandbox_run_body,
-        native_evaluate=True,
     )
 
 

examples/experimental/openenv/openenv_launch_common.py

@@ -222,15 +222,15 @@ def apply_optional_env_vars(env: dict[str, str], args: LaunchArgs) -> None:
             raise RuntimeError(
                 "the Daytona sandbox mode needs tbench2_env in the rollout "
                 "process's environment: pip install -e '<OpenEnv>/envs/tbench2_env' "
-                "from the pinned checkout in this directory's README"
+                "from the checkout described in this directory's README"
             ) from e
         server_src = Path(tbench2_env.__file__).resolve().parent / "server" / "tbench2_env_environment.py"
         src_text = server_src.read_text(encoding="utf-8") if server_src.is_file() else ""
         if "TB2_WITHHOLD_TESTS" not in src_text:
             raise RuntimeError(
                 "the installed tbench2_env server lacks the native-evaluate "
                 "contract (canonical test.sh scoring / TB2_WITHHOLD_TESTS): "
-                "install the pinned checkout from this directory's README, "
-                "not upstream main"
+                "install from an OpenEnv checkout at or after the #1012 merge "
+                "(04d259ea6) — see this directory's README"
             )
         env["OPENENV_TB2_TASKS_DIR"] = args.openenv_tb2_tasks_dir

examples/experimental/openenv/scan_golden.py

@@ -93,16 +93,26 @@ async def run() -> dict:
                 t = time.monotonic()
                 res = await env.step(action(action_type="evaluate"))
                 m = {
-                    "reward": float(getattr(res, "reward", 0.0) or 0.0),
                     "solve_exit": solve_exit,
                     "solve_s": round(solve_s, 1),
                     "eval_s": round(time.monotonic() - t, 1),
                 }
-                if capture_logs and m["reward"] < 1.0:
-                    # The native-evaluate server's output carries the test.sh
-                    # log tail itself; the on-disk copy lives under
-                    # /logs/verifier only for the verify window (no more
-                    # /tmp/tb2_testsh.log to read back).
+                # Same no-verdict semantics as the agent loop's guard: a
+                # server-side scoring failure or a non-canonical harness is
+                # reported as ERR, not as a fake 0.0 that would misattribute
+                # an infra problem to the task.
+                raw_reward = getattr(res, "reward", None)
+                eval_error = oaf._obs_field(res, "error")
+                harness = str(oaf._obs_info(res).get("harness", ""))
+                if raw_reward is None or eval_error or harness != "tests/test.sh":
+                    m["reward"] = None
+                    m["error"] = f"no canonical verdict (error={eval_error!r}, harness={harness!r})"
+                else:
+                    m["reward"] = float(raw_reward)
+                if capture_logs and (m["reward"] is None or m["reward"] < 1.0):
+                    # The server's evaluate output carries the test.sh log
+                    # tail; the on-disk copy lives under /logs/verifier only
+                    # for the verify window.
                     m["test_log_tail"] = (oaf._obs_field(res, "output") or "")[-800:]
                     res = await env.step(action(action_type="exec", command="tail -c 1200 /tmp/solve.log 2>&1"))
                     m["solve_log_tail"] = oaf._obs_field(res, "output")

examples/experimental/openenv/tests/test_openenv_agent_function.py

@@ -33,14 +33,15 @@ def __init__(self, **kw):
 
 
 class _FakeResult:
-    def __init__(self, output="", reward=None, instruction=""):
-        self.observation = _FakeObs(output=output, instruction=instruction)
+    def __init__(self, output="", reward=None, instruction="", info=None):
+        self.observation = _FakeObs(output=output, instruction=instruction, info=info or {})
         if reward is not None:
             self.reward = reward
 
 
 class _FakeEnv:
-    """Records every step() action; answers both scoring protocols."""
+    """Records every step() action; answers `evaluate` like a contract-carrying
+    server (reward plus the canonical-harness marker)."""
 
     last_actions: list = []
 
@@ -60,9 +61,7 @@ async def reset(self, task_id=None):
     async def step(self, action):
         self.actions.append(action)
         if action.action_type == "evaluate":
-            return _FakeResult(reward=1.0)
-        if "test.sh" in (action.command or ""):
-            return _FakeResult(output=f"{oaf._REWARD_MARKER}1.0")
+            return _FakeResult(reward=1.0, info={"tests_passed": True, "harness": "tests/test.sh"})
         return _FakeResult(output="ok")
 
 
@@ -95,8 +94,10 @@ async def _create(self, **kw):
 
 
 def test_shared_leg_dispatch(monkeypatch):
-    """The shared-server run_episode: exec prefixed with the task workdir,
-    canonical-exec scoring, rm-hack present, standard `evaluate` never used."""
+    """The shared-server run_episode: exec commands pass through unmodified
+    (the server resolves the workdir), scoring via the standard `evaluate`
+    action — and the trial-dir purge (post_episode) runs, since the shared
+    server outlives the episode."""
     monkeypatch.setattr(oaf, "_load_tbench2", lambda: _CLASSES)
 
     async def spying_with_env(env_cls, env_url, body):
@@ -111,8 +112,35 @@ async def spying_with_env(env_cls, env_url, body):
     execs = [a for a in actions if a.action_type == "exec"]
 
     assert reward == 1.0
-    assert execs[0].command == "cd /app && echo hi"
-    assert any("bash /tests/test.sh" in a.command for a in execs)
-    assert any("/tmp/tbench2_env_runs" in a.command for a in execs), "rm-hack missing"
-    assert not any(a.action_type == "evaluate" for a in actions)
+    assert execs[0].command == "echo hi"
+    assert any("/tmp/tbench2_env_runs" in (a.command or "") for a in execs), "trial-dir purge missing"
+    assert any(a.action_type == "evaluate" for a in actions)
     assert metrics["turns"] == 2 and metrics["tool_calls"] == 1
+
+
+def test_old_server_reward_is_not_trusted(monkeypatch):
+    """A server without the canonical contract (e.g. an out-of-date install)
+    answers `evaluate` with a plausible-looking reward but no harness marker
+    (its info is {tests_passed, exit_code} from bare pytest). That reward must
+    be dropped, not ingested: source preflight is impossible against a remote
+    server."""
+
+    class _OldServerEnv(_FakeEnv):
+        async def step(self, action):
+            if action.action_type == "evaluate":
+                self.actions.append(action)
+                return _FakeResult(reward=1.0, info={"tests_passed": True, "exit_code": 0})
+            return await super().step(action)
+
+    monkeypatch.setattr(oaf, "_load_tbench2", lambda: {"env": _OldServerEnv, "action": _CLASSES["action"]})
+
+    async def spying_with_env(env_cls, env_url, body):
+        return await body(env_cls())
+
+    monkeypatch.setattr(oaf, "_with_env", spying_with_env)
+
+    reward, metrics = run_async(
+        oaf.run_episode(_FakePolicy(), "m", [{"role": "system", "content": "s"}], {}, {"task_id": "t1"})
+    )
+    assert reward is None
+    assert metrics["turns"] == 2  # the episode itself completed; only scoring was rejected

examples/experimental/openenv/tests/test_openenv_daytona_agent_function.py

@@ -6,8 +6,8 @@
     pytest examples/experimental/openenv/tests/ -q
 
 Covers what a live episode cannot cheaply prove:
-  - episode dispatch: this module's run_episode sends raw exec commands and
-    scores via the standard `evaluate` action;
+  - episode dispatch: this module's run_episode passes exec commands through
+    unmodified and scores via the standard `evaluate` action;
   - sandbox-create throttling: Daytona rate-limit errors are retried with
     backoff and a bounded budget, anything else propagates immediately; a
     cancel mid-create reaps the orphaned sandbox instead of leaking it.
@@ -38,8 +38,10 @@ def run_async(coro):
 
 
 def test_daytona_leg_dispatch(monkeypatch):
-    """The daytona run_episode: exec raw (server resolves the workdir), scoring
-    via the standard `evaluate` action, no canonical exec, no rm-hack."""
+    """The daytona run_episode: exec commands pass through unmodified (the
+    server resolves the workdir), scoring via the standard `evaluate` action —
+    and unlike the shared leg, no trial-dir purge (the sandbox is deleted
+    when the episode ends)."""
     monkeypatch.setattr(oaf, "_load_tbench2", lambda: _CLASSES)
 
     @asynccontextmanager
@@ -57,7 +59,6 @@ async def fake_episode_env(env_cls, metadata):
     assert reward == 1.0
     assert execs[0].command == "echo hi"
     assert any(a.action_type == "evaluate" for a in actions)
-    assert not any("test.sh" in (a.command or "") for a in execs)
     assert not any("/tmp/tbench2_env_runs" in (a.command or "") for a in execs)
     assert metrics["turns"] == 2 and metrics["tool_calls"] == 1