diff --git a/examples/experimental/openenv/README.md b/examples/experimental/openenv/README.md index 131bb7e26f8..496ae852c5f 100644 --- a/examples/experimental/openenv/README.md +++ b/examples/experimental/openenv/README.md @@ -51,11 +51,11 @@ few tasks at a time if it does. Whichever provider you pick, install `tbench2_env` **editable**: the recipe bakes the installed source into each task image, so that install must carry -the `>=` #1012 server contract. The launcher preflights the installed source +the `>=` #1025 server contract. The launcher preflights the installed source and fails fast on an older one. ```bash -git clone https://github.com/huggingface/OpenEnv.git # >= the #1012 merge (04d259ea6, the full canonical contract for both modes); pin that sha if you need frozen reward semantics across a long run +git clone https://github.com/huggingface/OpenEnv.git # >= the #1025 merge (38b2a3135: canonical contract for both modes, and a missing verdict is an error rather than reward 0); pin that sha if you need frozen reward semantics across a long run pip install -e OpenEnv/envs/tbench2_env ``` @@ -159,7 +159,7 @@ launcher uses `--openenv-env-url` instead. `MAX_CONCURRENT_ENVS` caps live containers; keep it at or below the rollout batch concurrency. Those containers are heavy on disk, so if you'd rather not colocate them with the GPU workload, run the server on a separate Docker host and point the launcher at it with -`--openenv-env-url http://:8003`. The same `>=` #1012 `tbench2_env` +`--openenv-env-url http://:8003`. The same `>=` #1025 `tbench2_env` contract applies: the adapter drops every episode (with a warning) from a server that doesn't carry it. diff --git a/examples/experimental/openenv/openenv_agent_function.py b/examples/experimental/openenv/openenv_agent_function.py index c0d01c999e6..89a16e4aad3 100644 --- a/examples/experimental/openenv/openenv_agent_function.py +++ b/examples/experimental/openenv/openenv_agent_function.py @@ -28,9 +28,10 @@ MILES_ROUTER_EXTERNAL_HOST optional host rewrite for off-cluster agents Server contract: the env server must run tbench2_env at or after the -huggingface/OpenEnv#1012 merge (04d259ea6; install per the README) — +huggingface/OpenEnv#1025 merge (38b2a3135; install per the README) — canonical tests/test.sh scoring inside the standard ``evaluate`` action, task -WORKDIR resolved server-side, verifier assets withheld. The adapter verifies +WORKDIR resolved server-side, verifier assets withheld, and a verifier that +never writes its verdict reported as a scoring error rather than reward 0. The adapter verifies the contract on every episode rather than trusting the deployment: an ``evaluate`` reply without the canonical-harness marker is treated as no verdict and the episode is dropped with a warning (see the guard in @@ -338,7 +339,8 @@ async def body(env: Any) -> tuple[float | None, int, str, list[float], list[floa # No canonical verdict -> reward None (the training wrapper drops the # sample instead of ingesting a false-negative 0): # - reward=None / `error` set: the scoring step itself errored - # server-side (toolkit timeout, staging I/O) -- not tests failing. + # server-side (toolkit timeout, staging I/O, or test.sh never + # wrote reward.txt) -- not tests failing. # - harness marker absent: the server scored, but not through the # canonical tests/test.sh (a tbench2_env install predating the # contract in the module docstring, or a task dir without test.sh diff --git a/examples/experimental/openenv/openenv_daytona_agent_function.py b/examples/experimental/openenv/openenv_daytona_agent_function.py index 5c01c0976ad..2a592e3f2c4 100644 --- a/examples/experimental/openenv/openenv_daytona_agent_function.py +++ b/examples/experimental/openenv/openenv_daytona_agent_function.py @@ -59,7 +59,7 @@ # creates hit Daytona's build cache, and no named snapshot is involved. # # The sandbox's env server is the tbench2_env baked by the recipe, installed -# per the README (at or after the huggingface/OpenEnv#1012 merge): canonical +# per the README (at or after the huggingface/OpenEnv#1025 merge): canonical # tests/test.sh scoring built into `evaluate`, task WORKDIR resolved # server-side, verifier assets withheld. The launcher preflight rejects an # older install outright, and the shared agent loop's harness-marker guard diff --git a/examples/experimental/openenv/openenv_e2b_agent_function.py b/examples/experimental/openenv/openenv_e2b_agent_function.py index eccc65ea6fb..b558f781aad 100644 --- a/examples/experimental/openenv/openenv_e2b_agent_function.py +++ b/examples/experimental/openenv/openenv_e2b_agent_function.py @@ -51,7 +51,7 @@ # The sandbox's env server is the tbench2_env baked by the recipe, installed -# per the README (at or after the huggingface/OpenEnv#1012 merge): canonical +# per the README (at or after the huggingface/OpenEnv#1025 merge): canonical # tests/test.sh scoring built into `evaluate`, task WORKDIR resolved # server-side, verifier assets withheld. The launcher preflight rejects an # older install outright, and the shared agent loop's harness-marker guard diff --git a/examples/experimental/openenv/openenv_launch_common.py b/examples/experimental/openenv/openenv_launch_common.py index 3f96eb412d8..cf9f359ad95 100644 --- a/examples/experimental/openenv/openenv_launch_common.py +++ b/examples/experimental/openenv/openenv_launch_common.py @@ -232,12 +232,17 @@ def apply_optional_env_vars(env: dict[str, str], args: LaunchArgs) -> None: ) from e server_src = Path(tbench2_env.__file__).resolve().parent / "server" / "tbench2_env_environment.py" src_text = server_src.read_text(encoding="utf-8") if server_src.is_file() else "" - if "TB2_WITHHOLD_TESTS" not in src_text: + # `_require_canonical_verdict` (#1025) is what turns a verifier that never + # wrote reward.txt into an error; before it, that reply was reward 0.0 + # WITH the harness marker, which the per-episode guard cannot tell from + # a genuine failure. + if "TB2_WITHHOLD_TESTS" not in src_text or "_require_canonical_verdict" not in src_text: raise RuntimeError( "the installed tbench2_env server lacks the native-evaluate " - "contract (canonical test.sh scoring / TB2_WITHHOLD_TESTS): " - "install from an OpenEnv checkout at or after the #1012 merge " - "(04d259ea6) — see this directory's README" + "contract (canonical test.sh scoring / TB2_WITHHOLD_TESTS / " + "missing verdict reported as an error): install from an OpenEnv " + "checkout at or after the #1025 merge (38b2a3135) — see this " + "directory's README" ) env["OPENENV_TB2_TASKS_DIR"] = args.openenv_tb2_tasks_dir diff --git a/examples/experimental/openenv/openenv_modal_agent_function.py b/examples/experimental/openenv/openenv_modal_agent_function.py index 6c87a641406..2e5d32bc8b6 100644 --- a/examples/experimental/openenv/openenv_modal_agent_function.py +++ b/examples/experimental/openenv/openenv_modal_agent_function.py @@ -53,7 +53,7 @@ # The sandbox's env server is the tbench2_env baked by the recipe, installed -# per the README (at or after the huggingface/OpenEnv#1012 merge): canonical +# per the README (at or after the huggingface/OpenEnv#1025 merge): canonical # tests/test.sh scoring built into `evaluate`, task WORKDIR resolved # server-side, verifier assets withheld. The launcher preflight rejects an # older install outright, and the shared agent loop's harness-marker guard