feat: add taskset selection views and select config

configs/agentic_judge.toml

@@ -1,4 +1,5 @@
-num_tasks = 2
+[select]
+limit = 2
 
 [env]
 id = "agentic-judge"

configs/alphabet_sort.toml

@@ -1,9 +1,11 @@
 # alphabet-sort — multi-turn alphabetical sorting, driven by a colocated user simulator.
 #
 #   uv run vf-eval @ configs/alphabet_sort.toml --model <small-instruct-model>
-num_tasks = 5
 num_rollouts = 2
 
+[select]
+limit = 5
+
 [env.taskset]
 id = "alphabet-sort"
 min_turns = 2

configs/code_golf.toml

@@ -3,8 +3,10 @@
 # each other (shortest wins), so one rollout per task is a full comparison.
 #
 #   uv run vf-eval code-golf @ configs/code_golf.toml
-num_tasks = 1
 num_rollouts = 1
 
+[select]
+limit = 1
+
 [env.taskset]
 id = "code-golf"

configs/deepwiki.toml

@@ -1,7 +1,9 @@
 # deepwiki — an existing remote MCP server, reached by URL (nothing to run ourselves).
 #
 #   uv run vf-eval @ configs/deepwiki.toml
-num_tasks = 1
+
+[select]
+limit = 1
 
 [env.taskset]
 id = "deepwiki"

configs/glossary.toml

@@ -1,7 +1,9 @@
 # glossary — a colocated MCP tool server (runs inside the harness's own runtime).
 #
 #   uv run vf-eval glossary @ configs/glossary.toml
-num_tasks = 1
+
+[select]
+limit = 1
 
 [env.taskset]
 id = "glossary"

configs/gsm8k.toml

@@ -1,8 +1,10 @@
 # gsm8k — grade-school math, bash harness; a few tasks with several rollouts each.
 #
 #   uv run vf-eval gsm8k @ configs/gsm8k.toml
-num_tasks = 5
 num_rollouts = 3
 
+[select]
+limit = 5
+
 [env.taskset]
 id = "gsm8k"

configs/gsm8k_rlm.toml

@@ -7,7 +7,9 @@
 # file fills the typed config, each side validated against its specific type (GSM8KConfig /
 # RLMHarnessConfig). prime-rl consumes the same [env] shape, reading the ids straight from
 # its config (TOML-driven there too).
-num_tasks = 1
+
+[select]
+limit = 1
 
 [env.taskset]
 id = "gsm8k"

configs/harbor.toml

@@ -2,7 +2,9 @@
 # The `bash` harness gives the agent the bash (+ edit) tools it drives the terminal with.
 #
 #   uv run vf-eval @ configs/harbor.toml
-num_tasks = 10
+
+[select]
+limit = 10
 
 [env.taskset]
 id = "harbor"

configs/textarena.toml

@@ -4,9 +4,11 @@
 # the harness only ever sees the conversation.
 #
 #   uv run vf-eval @ configs/textarena.toml --model <small-instruct-model>
-num_tasks = 5
 num_rollouts = 2
 
+[select]
+limit = 5
+
 [env.taskset]
 id = "textarena"
 game = "Wordle-v0"

configs/wiki_search.toml

@@ -1,7 +1,9 @@
 # wiki-search — a shared MCP tool server (the corpus is built once, reused across rollouts).
 #
 #   uv run vf-eval @ configs/wiki_search.toml
-num_tasks = 1
+
+[select]
+limit = 1
 
 [env.taskset]
 id = "wiki-search"

configs/wiki_search_compact.toml

@@ -2,7 +2,9 @@
 # so the model always sees a compacted context instead of the full history.
 #
 #   uv run vf-eval @ configs/wiki_search_compact.toml
-num_tasks = 1
+
+[select]
+limit = 1
 
 [env.taskset]
 id = "wiki-search"

configs/wordle.toml

@@ -1,9 +1,11 @@
 # wordle — the textarena taskset pinned to Wordle, the engine playing the user.
 #
 #   uv run vf-eval @ configs/wordle.toml --model <small-instruct-model>
-num_tasks = 5
 num_rollouts = 2
 
+[select]
+limit = 5
+
 [env.taskset]
 id = "wordle"
 

docs/v1/evaluation.md

@@ -37,11 +37,11 @@ The output from evaluations are written into `outputs/<env>--<model>--<harness>/
 - `sampling` — generation params passed to the model, e.g. `sampling.temperature`
 - `env.taskset.id` — pick the taskset (or the positional `eval <taskset-id>`)
 - `env.agent.harness.id` — pick the agent's harness (`[env.agent.harness]` in TOML)
-- `num_tasks` — how many tasks to evaluate. Not setting a value means all tasks; an
-  infinite taskset (a procedural generator, e.g. `wordle`) requires it
+- `select` — which tasks to evaluate (see [Selecting tasks](tasksets.md#selecting-tasks)).
+  `-n` sets `select.limit`: not setting it means all tasks, and an infinite taskset (a
+  procedural generator, e.g. `wordle`) requires it. `-s` sets `select.shuffle` (fixed seed)
 - `num_rollouts` — rollouts per task
 - `verbose` — log at debug instead of info
-- `shuffle` — samples the task order (fixed seed); an error on an infinite taskset
 - `rich` — the live dashboard (default); `--no-rich` streams logs to the console and
   prints each trace as JSON at the end
 - `rich.show_logs` — replace the dashboard's per-rollout rows with a live tail of the

docs/v1/tasksets.md

@@ -65,7 +65,7 @@ class AdditionTaskset(vf.Taskset[AdditionTask, AdditionConfig]):
     def load(self) -> list[AdditionTask]:
         return [
             AdditionTask(
-                AdditionData(idx=i, prompt=f"What is {i} + {i}?", answer=2 * i),
+                AdditionData(prompt=f"What is {i} + {i}?", answer=2 * i),
                 self.config.task,
             )
             for i in range(self.config.num_tasks)
@@ -112,11 +112,41 @@ These values can be overridden with `--env.taskset.num-tasks` and `--env.taskset
 
 ## Task identity
 
-A task's `hash` identifies its exact serialized data. Its `key` provides stable identity across runs and defaults to the hash. Override `Task.key` with a durable source ID when task data contains run-local fields such as an autogenerated `idx`; keys must be unique within a taskset. Both values are recorded on traces.
+A task's `idx` is its index in the `load()` stream; it is set automatically. Set `TaskData.id` to a durable ID from the source (e.g. a dataset's instance ID) and `TaskData.name` to a readable name; both are optional.
+
+A task's `hash` identifies its exact serialized data. Its `key` provides stable identity across runs and defaults to the hash. Override `Task.key` with a durable source ID when task data contains run-local fields such as `idx`; keys must be unique within a taskset. Both values are recorded on traces.
+
+## Selecting tasks
+
+Views on a taskset pick which tasks an iteration yields. They are lazy and chain in any order:
+
+```python
+taskset.include(idx=["0:100"]).exclude(names=["broken"]).shuffle(seed=0).take(5)
+```
+
+`include` keeps and `exclude` drops the tasks named by `idx` (ints and Python slices `start:stop:step`, e.g. `"100:"`, `":50"`, `"::2"`; no negative positions), `ids`, `keys` or `names`; a task matches when any list names it. An entry that matches no task logs a warning once the stream was read.
+
+The eval, debug, validate and GEPA entrypoints take a `select` block and apply it with `taskset.select(config.select)`, which chains the views in a fixed order: `include`, `exclude`, `shuffle`, `skip`, `limit`:
+
+```bash
+uv run vf-eval gsm8k --select.limit 50 --select.shuffle         # 50 random tasks (short: -n 50 -s)
+uv run vf-eval gsm8k --select.include.idx 0:100                 # the first 100 tasks
+uv run vf-eval gsm8k --select.shuffle --select.skip 100 --select.limit 100   # a random split disjoint from the first
+```
+
+```toml
+[select]
+include = { idx = ["0:500"] }
+exclude = { names = ["broken-task"] }
+shuffle = true
+limit = 100
+```
+
+Because the shuffle comes before `skip` and `limit`, a larger `limit` extends the same selection instead of drawing a new one.
 
 ## Lazy and infinite tasksets
 
-`load()` may be a generator instead of returning a list: yield each task as it's built. Consumers iterate the taskset lazily (`Taskset.head` pulls only what a run needs — `eval -n 5` builds 5 tasks, not the whole set) — so a generator pays off whenever building a task is expensive.
+`load()` may be a generator instead of returning a list: yield each task as it's built. Consumers iterate the taskset lazily (`eval -n 5` builds 5 tasks, not the whole set, unless `shuffle` needs all of them) — so a generator pays off whenever building a task is expensive.
 
 A procedural taskset can keep yielding forever. Declare `INFINITE = True` so consumers know the stream never ends — infinity is inherent to the taskset, not a config knob; how many tasks a run takes is the run's choice (`-n`), not the taskset's:
 
@@ -131,12 +161,12 @@ class AdditionTaskset(vf.Taskset[AdditionTask, vf.TasksetConfig]):
     def load(self) -> Iterator[AdditionTask]:
         for i in itertools.count():
             yield AdditionTask(
-                AdditionData(idx=i, prompt=f"What is {i} + {i}?", answer=2 * i),
+                AdditionData(prompt=f"What is {i} + {i}?", answer=2 * i),
                 self.config.task,
             )
 ```
 
-Two rules follow from infinity: a run over an infinite taskset must be bounded with `num_tasks` (`-n` on the CLI — omitting it is an error), and `shuffle` is an error: there is no whole set to sample from — bound the stream first (`taskset.head(n).shuffle()`). The generator runs once, client-side (the eval entrypoint or the prime-rl orchestrator pulls tasks off it and ships each task's data to the env server), so nothing needs to re-produce the same sequence across processes; keep `load()` deterministic only if you want `--resume` to regenerate the same first `n` tasks (see `alphabet_sort`, `color_codeword`, or the built-in `textarena` taskset).
+Two rules follow from infinity: a run over an infinite taskset must be bounded with `take` (`-n` on the CLI) or an `include` of only closed `idx` ranges — omitting both is an error — and `shuffle` needs a bound before it: there is no whole set to sample from otherwise. `select` applies `limit` after the shuffle, so on the CLI only closed `include.idx` ranges can bound it (`--select.include.idx 0:1000 -s -n 50`); in Python, `take(n).shuffle()` works too. The generator runs once, client-side (the eval entrypoint or the prime-rl orchestrator pulls tasks off it and ships each task's data to the env server), so nothing needs to re-produce the same sequence across processes; keep `load()` deterministic only if you want `--resume` to regenerate the same first `n` tasks (see `alphabet_sort`, `color_codeword`, or the built-in `textarena` taskset).
 
 ## Adding Tools
 
@@ -227,7 +257,7 @@ class JudgeTraceTaskset(vf.Taskset[JudgedTask, SetConfig]):
     def load(self) -> list[JudgedTask]:
         return [
             JudgedTask(
-                JudgedData(idx=0, prompt="What is 2+2?", answer="4"),
+                JudgedData(prompt="What is 2+2?", answer="4"),
                 self.config.task,
             )
         ]

environments/alphabet_sort/alphabet_sort/taskset.py

@@ -193,7 +193,6 @@ def sort_key(s: str, by_first: bool = by_first) -> str:
 
                 yield AlphabetSortTask(
                     AlphabetSortTaskData(
-                        idx=idx,
                         # No prompt on the row: the scripted user opens with the sort prompt,
                         # then the follow-ups — one user turn per `user_turns` entry.
                         prompt=None,

environments/bash_interception/bash_interception/taskset.py

@@ -86,6 +86,6 @@ def load(self) -> list[BashInterceptionTask]:
             ),
         )
         return [
-            BashInterceptionTask(vf.TaskData(idx=i, prompt=prompt), self.config.task)
-            for i, prompt in enumerate(prompts)
+            BashInterceptionTask(vf.TaskData(prompt=prompt), self.config.task)
+            for prompt in prompts
         ]

environments/code_golf/code_golf/taskset.py

@@ -114,12 +114,11 @@ def load(self) -> list[CodeGolfTask]:
         return [
             CodeGolfTask(
                 CodeGolfData(
-                    idx=i,
                     name=name,
                     prompt=f"{SYSTEM}\n\nPrint {description}.",
                     expected=expected,
                 ),
                 self.config.task,
             )
-            for i, (name, description, expected) in enumerate(self.SPECS)
+            for name, description, expected in self.SPECS
         ]

environments/color_codeword/color_codeword/taskset.py

@@ -9,7 +9,6 @@
 the v1 message graph as `mm_kwargs` for training.
 """
 
-import itertools
 import random
 import re
 from collections.abc import Iterator
@@ -158,7 +157,7 @@ def load(self) -> Iterator[ColorCodewordTask]:
             for color in colors
         }
         length = c.images_per_turn * MAX_TURNS
-        for idx in itertools.count():
+        while True:
             sequence = [rng.choice(colors) for _ in range(length)]
             answer = "".join(COLOR_MAP[col] for col in sequence)
             colors_per_turn = [
@@ -173,7 +172,6 @@ def load(self) -> Iterator[ColorCodewordTask]:
             ] + [vf.TextContentPart(text=text)]
             yield ColorCodewordTask(
                 ColorCodewordTaskData(
-                    idx=idx,
                     prompt=[vf.UserMessage(content=parts)],
                     system_prompt=SYSTEM_PROMPT,
                     answer=answer,

environments/deepwiki/deepwiki/taskset.py

@@ -45,7 +45,6 @@ def load(self) -> list[DeepWikiTask]:
         return [
             DeepWikiTask(
                 DeepWikiTaskData(
-                    idx=i,
                     name=repo,
                     prompt=(
                         f"Use the `deepwiki_ask_question` tool to ask what programming "
@@ -56,5 +55,5 @@ def load(self) -> list[DeepWikiTask]:
                 ),
                 self.config.task,
             )
-            for i, (repo, language) in enumerate(TASKS)
+            for repo, language in TASKS
         ]

environments/glossary/glossary/taskset.py

@@ -39,7 +39,6 @@ def load(self) -> list[GlossaryTask]:
         return [
             GlossaryTask(
                 GlossaryTaskData(
-                    idx=i,
                     name=entity.title(),
                     prompt=(
                         f'Use the `facts_lookup` tool to look up "{entity.title()}", then '
@@ -49,5 +48,5 @@ def load(self) -> list[GlossaryTask]:
                 ),
                 self.config.task,
             )
-            for i, (entity, fact) in enumerate(FACTS.items())
+            for entity, fact in FACTS.items()
         ]

environments/grayscale_interception/grayscale_interception/taskset.py

@@ -81,8 +81,4 @@ def load(self) -> list[GrayscaleInterceptionTask]:
                 ]
             )
         ]
-        return [
-            GrayscaleInterceptionTask(
-                vf.TaskData(idx=0, prompt=prompt), self.config.task
-            )
-        ]
+        return [GrayscaleInterceptionTask(vf.TaskData(prompt=prompt), self.config.task)]

environments/gsm8k/gsm8k/taskset.py

@@ -68,11 +68,10 @@ def load(self) -> list[GSM8KTask]:
         return [
             GSM8KTask(
                 GSM8KData(
-                    idx=i,
                     prompt=f"{SYSTEM}\n\n{row['question']}",
                     answer=row["answer"].split("####")[-1].strip(),
                 ),
                 self.config.task,
             )
-            for i, row in enumerate(rows)
+            for row in rows
         ]

environments/interception/interception/taskset.py

@@ -66,8 +66,8 @@ def load(self) -> list[InterceptionTask]:
         markers = ("DETERMINISTIC_BLOCK", "JUDGE_BLOCK", "STOP", "METRIC_ONLY")
         return [
             InterceptionTask(
-                vf.TaskData(idx=index, prompt=f"Reply with exactly {marker}."),
+                vf.TaskData(prompt=f"Reply with exactly {marker}."),
                 self.config.task,
             )
-            for index, marker in enumerate(markers)
+            for marker in markers
         ]

environments/kuhn_poker/kuhn_poker/taskset.py

@@ -162,5 +162,5 @@ class KuhnPokerTaskset(vf.Taskset[KuhnPokerTask, KuhnPokerConfig]):
     def load(self) -> Iterator[KuhnPokerTask]:
         for i in count():
             yield KuhnPokerTask(
-                KuhnPokerData(idx=i, name=f"hand#{i}", prompt=None, info={"seed": i})
+                KuhnPokerData(name=f"hand#{i}", prompt=None, info={"seed": i})
             )

environments/proposer_solver/proposer_solver/taskset.py

@@ -182,12 +182,11 @@ def load(self) -> list[ProposeTask]:
         return [
             ProposeTask(
                 SeedData(
-                    idx=i,
                     name=topic.replace(" ", "-"),
                     prompt=PROPOSE.format(topic=topic),
                     topic=topic,
                 ),
                 self.config.task,
             )
-            for i, topic in enumerate(self.TOPICS)
+            for topic in self.TOPICS
         ]

environments/reverse_text/reverse_text/taskset.py

@@ -49,12 +49,11 @@ def load(self) -> list[ReverseTextTask]:
         return [
             ReverseTextTask(
                 ReverseTextData(
-                    idx=i,
                     prompt=row["prompt"],
                     system_prompt=SYSTEM,
                     answer=row["prompt"][::-1],
                 ),
                 self.config.task,
             )
-            for i, row in enumerate(rows)
+            for row in rows
         ]

environments/scratchpad/scratchpad/taskset.py

@@ -56,11 +56,10 @@ def load(self) -> list[ScratchpadTask]:
         return [
             ScratchpadTask(
                 ScratchpadTaskData(
-                    idx=i,
                     word=w,
                     prompt=INSTRUCTION.format(word=w),
                 ),
                 self.config.task,
             )
-            for i, w in enumerate(WORDS)
+            for w in WORDS
         ]