feat: first-class v1 harnesses/tasksets packages (+ README, serve/CLI fixes)

configs/harbor.toml

@@ -1,11 +1,11 @@
-# harbor — Terminal-Bench 2, each task run in its own declared container image (needs docker).
+# harbor-v1 — Terminal-Bench 2, each task run in its own declared container image (needs docker).
 # enable_bash turns on the shell tool the agent drives the terminal with.
 #
 #   uv run eval @ configs/harbor.toml
 num_tasks = 10
 
 [taskset]
-id = "harbor"
+id = "harbor-v1"
 dataset = "terminal-bench/terminal-bench-2"
 
 [harness]

examples/tasksets/terminal_bench_2_v1/terminal_bench_2_v1/__init__.py

@@ -1,12 +1,12 @@
-"""terminal-bench-2-v1 — the harbor taskset pinned to Terminal-Bench 2 (example env).
+"""terminal-bench-2-v1 — the harbor-v1 taskset pinned to Terminal-Bench 2 (example env).
 
-A thin wrapper over `harbor`: pins `dataset` to "terminal-bench/terminal-bench-2". Needs the
-`harbor` CLI (`uv tool install harbor`) and a container runtime (docker/prime), like harbor.
+A thin wrapper over `harbor-v1`: pins `dataset` to "terminal-bench/terminal-bench-2". Needs
+the `harbor` CLI (`uv tool install harbor`) and a container runtime (docker/prime).
 """
 
 from typing import Literal
 
-from harbor import HarborConfig, HarborTaskset
+from tasksets.harbor_v1 import HarborConfig, HarborTaskset
 
 
 class TerminalBench2Config(HarborConfig):

examples/tasksets/wordle_v1/wordle_v1/__init__.py

@@ -7,7 +7,7 @@
 
 from typing import Literal
 
-from textarena_v1 import TextArenaConfig, TextArenaTaskset
+from tasksets.textarena_v1 import TextArenaConfig, TextArenaTaskset
 
 
 class WordleConfig(TextArenaConfig):

packages/harnesses/harnesses/__init__.py

@@ -0,0 +1,13 @@
+"""Built-in harnesses, resolved by id (`--harness.id <id>`) as `harnesses.<id>`.
+
+Re-exports each harness's class + config off the package."""
+
+from harnesses.default import DefaultHarness, DefaultHarnessConfig
+from harnesses.rlm import RLMHarness, RLMHarnessConfig
+
+__all__ = [
+    "DefaultHarness",
+    "DefaultHarnessConfig",
+    "RLMHarness",
+    "RLMHarnessConfig",
+]

packages/harnesses/default/__init__.py → packages/harnesses/harnesses/default/__init__.py

@@ -1,7 +1,7 @@
 """default — v1's built-in harness: its `harness.py` (class + config) and the
 `program.py` script it stages into the runtime. Resolved by id via `load_harness`."""
 
-from default.harness import DefaultHarness, DefaultHarnessConfig
+from harnesses.default.harness import DefaultHarness, DefaultHarnessConfig
 
 
 def load_harness(config: DefaultHarnessConfig) -> DefaultHarness:

packages/harnesses/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "harnesses"
 version = "0.1.0"
-description = "Shipped v1 harnesses (resolved by id via `import <id>`), vendored by default."
+description = "Built-in harnesses"
 requires-python = ">=3.10"
 dependencies = []
 
@@ -10,4 +10,4 @@ requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
-packages = ["default", "rlm"]
+packages = ["harnesses"]

packages/tasksets/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "tasksets"
 version = "0.1.0"
-description = "Shipped v1 tasksets (resolved by id via `import <id>`), vendored by default."
+description = "Built-in tasksets"
 requires-python = ">=3.10"
 dependencies = []
 
@@ -15,4 +15,4 @@ requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
-packages = ["harbor", "textarena_v1"]
+packages = ["tasksets"]

packages/tasksets/tasksets/__init__.py

@@ -0,0 +1,9 @@
+"""Built-in tasksets, resolved by id (`--taskset.id <id>`) as `tasksets.<id>`.
+
+Re-exports each taskset's class + config off the package. `textarena_v1` is not re-exported
+here — it imports the optional `textarena` dependency at module load — but still resolves as
+`tasksets.textarena_v1`."""
+
+from tasksets.harbor_v1 import HarborConfig, HarborTaskset
+
+__all__ = ["HarborConfig", "HarborTaskset"]

packages/tasksets/textarena_v1/__init__.py → packages/tasksets/tasksets/textarena_v1/__init__.py

@@ -94,7 +94,7 @@ def observation(seed: int) -> str:
     def user(self, task: TextArenaTask) -> vf.User:
         return vf.User(
             name="user",
-            command=[sys.executable, "-m", "textarena_v1.server"],
+            command=[sys.executable, "-m", "tasksets.textarena_v1.server"],
             env={"TEXTARENA_INFO": json.dumps(task.info)},
         )
 

packages/tasksets/textarena_v1/server.py → packages/tasksets/tasksets/textarena_v1/server.py

@@ -20,7 +20,7 @@
 
 import verifiers.v1 as vf
 
-from textarena_v1 import OUTCOME_FILE
+from tasksets.textarena_v1 import OUTCOME_FILE
 
 import nltk
 

pyproject.toml

@@ -57,7 +57,7 @@ dependencies = [
     "pymupdf",
     "pillow",
     "certifi",
-    "prime-pydantic-config[toml]",
+    "prime-pydantic-config[toml]>=0.3.0.dev86",
     "uvloop>=0.21.0; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'",
     # v1 (verifiers.v1) runtime — also used by the v0 -> vf.Trace bridge (verifiers.v1.legacy)
     "loguru>=0.7.0",
@@ -91,7 +91,7 @@ harnesses = ["harnesses"]
 examples = [
     "compact",
     "gsm8k-v1", "wikispeedia-v1", "glossary-v1", "deepwiki-v1", "wiki-search-v1", "code-golf-v1",
-    "reverse-text-v1", "hello-rlm-v1", "math-env-v1", "aime24-v1",
+    "reverse-text-v1", "math-env-v1", "aime24-v1",
     "wordle-v1", "terminal-bench-2-v1", "alphabet-sort-v1",
 ]
 
@@ -141,7 +141,6 @@ default = true
 tasksets = { path = "packages/tasksets", editable = true }
 harnesses = { path = "packages/harnesses", editable = true }
 compact = { path = "examples/harnesses/compact", editable = true }
-hello-rlm-v1 = { path = "examples/tasksets/hello_rlm_v1", editable = true }
 glossary-v1 = { path = "examples/tasksets/glossary_v1", editable = true }
 deepwiki-v1 = { path = "examples/tasksets/deepwiki_v1", editable = true }
 wikispeedia-v1 = { path = "examples/tasksets/wikispeedia_v1", editable = true }