feat: hardcode rollout filters and zero-advantage drop

README.md

@@ -217,7 +217,7 @@ Check out the [docs](docs) directory for in-depth guides on how to use prime-rl.
 - [**Configuration**](docs/configuration.md) - TOML composition, CLI overrides, env vars, validation
 - [**Training**](docs/training.md) - RL, SFT, evals, checkpointing, observability, rules of thumb
 - [**Scaling**](docs/scaling.md) - Single-GPU through multi-node, FSDP/EP/CP, SLURM, benchmarking
-- [**Algorithms**](docs/algorithms.md) - Async/off-policy training, the AIPO loss, advantage and filter plugins, trajectory merging
+- [**Algorithms**](docs/algorithms.md) - Async/off-policy training, advantage plugins, rollout checks, trajectory merging
 - [**Advanced**](docs/advanced.md) - Custom modeling, multimodal training, LoRA
 - [**Development**](docs/development.md) - Test suite, pre-commit hooks, adding a new model
 

configs/ci/nightly-fft/wiki-search.toml

@@ -17,11 +17,7 @@ name = "Qwen/Qwen3-4B-Instruct-2507"
 [orchestrator]
 batch_size = 512
 group_size = 16
-oversampling_factor = 2.0
-
-[[orchestrator.pre_batch_filters]]
-type = "zero_advantage"
-enforce = true
+max_inflight_episodes = 1024
 
 [[orchestrator.train.source]]
 name = "wiki-search"

configs/debug/algo/echo.toml

@@ -44,12 +44,6 @@ type = "subprocess"
 [orchestrator.train.sampling]
 max_completion_tokens = 512
 
-# ECHO learns from observation tokens even when the GRPO advantage collapses
-# to zero — keep zero-advantage rollouts in the batch.
-[[orchestrator.post_batch_filters]]
-type = "zero_advantage"
-enforce = false
-
 # Fine-tune inherits the PrimeIntellect Qwen3 template byte-for-byte.
 [orchestrator.renderer]
 name = "prime-qwen3"

docs/algorithms.md

@@ -1,6 +1,6 @@
 # Algorithms
 
-This page covers the math and the configurable algorithmic components: the algorithm abstraction and its algorithms, how off-policy training works, the loss components and advantage functions, how to plug in your own, the filters applied between rollout and training, and how multi-turn rollouts get merged into training samples.
+This page covers the math and the configurable algorithmic components: the algorithm abstraction and its algorithms, how off-policy training works, the loss components and advantage functions, how to plug in your own, the checks applied between rollout and training, and how multi-turn rollouts get merged into training samples.
 
 ## Table of Contents
 
@@ -164,12 +164,12 @@ At runtime, each env's resolved config builds two objects: a `Sampler` (`prime_r
 | `hierarchical_grpo` | `HierarchicalGRPOAlgorithm` | `score_group`: GRPO baseline per episode for solvers, per group for the proposer |
 | `opd` | `OPDAlgorithm` | `score_rollout`: own-context prefill under the teacher |
 | `opsd` | `OPSDAlgorithm` | `score_rollout`: demo-conditioned prefill under the live policy |
-| `sft` | `SFTDistillAlgorithm` | `score_group`: group-norm credit (feeds filters) |
+| `sft` | `SFTDistillAlgorithm` | no credit — ce trains on every sampled token |
 
 Each class owns its hooks outright — reading one top to bottom reads the algorithm, and everything on the class is an override point. The two hooks are one scope-and-timing ladder — the wider scope is unlocked by a later barrier, so the two axes coincide. Each is handed the `Rollout` directly — the env's typed trace (`reward`, `nodes`, `num_turns`, ...) with `samples` attached, plus `assign_advantages` to write credit:
 
-- `async score_rollout(rollout)` — one rollout, **on arrival** (as it's tokenized, before its group is complete): rollout-local credit (`rollout.assign_advantages(...)`, scalar broadcast or per-token), observation ce weights, **or** model I/O — query a reference pool (e.g. `self.teacher_pool`, connected in `setup()` via `self.connect(...)`, or the live `self.policy_pool` for opsd) and attach per-token results (e.g. teacher logprobs) with bounded concurrency. No siblings. `echo` weights observation tokens here, identifying env-provided observation nodes by their non-sampled status and source step role attribution, applying the optional user filter, and writing the `ce_weights` stream. Model I/O runs *before* the pre-batch filters, so it pays compute on rollouts that may then be filtered out.
-- `score_group(group)` — the cohort, **before filtering** (filters read the streams), synchronous: group-relative credit (GRPO/MaxRL baselines). `group` is a list of `Rollout`.
+- `async score_rollout(rollout)` — one rollout, **on arrival** (as it's tokenized, before its group is complete): rollout-local credit (`rollout.assign_advantages(...)`, scalar broadcast or per-token), observation ce weights, **or** model I/O — query a reference pool (e.g. `self.teacher_pool`, connected in `setup()` via `self.connect(...)`, or the live `self.policy_pool` for opsd) and attach per-token results (e.g. teacher logprobs) with bounded concurrency. No siblings. `echo` weights observation tokens here, identifying env-provided observation nodes by their non-sampled status and source step role attribution, applying the optional user filter, and writing the `ce_weights` stream. Model I/O runs *before* the zero-advantage drop, so it pays compute on rollouts that may then be dropped.
+- `score_group(group)` — the cohort, **before the zero-advantage drop** (the check reads the streams), synchronous: group-relative credit (GRPO/MaxRL baselines). `group` is a list of `Rollout`.
 
 The pipeline drives the hooks through two non-virtual methods it never looks inside: `algorithm.finalize_rollout(rollout)` per arrival (rollout-local scoring + reference I/O) and `algorithm.finalize_group(rollouts)` per group (scoring + wire stamping; after this the records are frozen — groups die at stamping). Sample construction (interleaving) is pure pipeline — observation-token provenance is available through structural attribution (`node.sampled`, `node.is_content`) for any algorithm that trains on env-provided tokens.
 
@@ -305,8 +305,8 @@ The per-token training signal is set by `algo.type` and the [algorithm](#the-alg
 | `rae` | `rl` | Reward minus a per-agent EMA baseline (SPIRAL's role-conditioned advantage estimation) — for multi-agent self-play envs. |
 | `hierarchical_grpo` | `rl` | GRPO for proposer-solver envs: solvers are compared within one proposed problem, while proposers are compared across proposals. |
 | `echo` | `rl` + `ce` | Group-norm on action tokens, plus weighted CE on env-provided tokens selected by message role (each role's `alpha` is its ECHO λ), optionally narrowed by a user filter. |
-| `opd` | `ref_kl` | On-policy distillation: per-token reverse KL to a reference model (`teacher`, an inline frozen hosted model), evaluated in the trainer from shipped reference logprobs. No credit — rollouts keep `advantages = None` (advantage-based filters never fire) and ship no advantage stream; `group_size` only fans out sampling. |
-| `opsd` | `ref_kl` | SDFT: per-token reverse KL to a demo-conditioned reference. No credit — rollouts keep `advantages = None` (advantage-based filters never fire) and ship no advantage stream. |
+| `opd` | `ref_kl` | On-policy distillation: per-token reverse KL to a reference model (`teacher`, an inline frozen hosted model), evaluated in the trainer from shipped reference logprobs. No credit — rollouts keep `advantages = None` (the zero-advantage drop never fires) and ship no advantage stream; `group_size` only fans out sampling. |
+| `opsd` | `ref_kl` | SDFT: per-token reverse KL to a demo-conditioned reference. No credit — rollouts keep `advantages = None` (the zero-advantage drop never fires) and ship no advantage stream. |
 | `sft` | `ce` | Cross-entropy on the sampled tokens. Assigns no advantage — trains on every sampled token. |
 
 ### Default Advantage
@@ -382,7 +382,7 @@ id = "null"
 type = "subprocess"
 ```
 
-`group_size` controls how many problems are proposed from each source task. `env.n` controls how many solvers attempt each proposed problem. If a comparison contains only one trace—for example, a solver when `env.n = 1`—its advantage is zero and the zero-advantage filter removes it.
+`group_size` controls how many problems are proposed from each source task. `env.n` controls how many solvers attempt each proposed problem. If a comparison contains only one trace—for example, a solver when `env.n = 1`—its advantage is zero and the zero-advantage drop removes it.
 
 This algorithm is accepted only for proposer-solver envs. Use the env's `train_proposer` and `train_solver` settings if you want to train only one role.
 
@@ -439,7 +439,7 @@ class MyAlgorithm(Algorithm):
 
 Add a typed `MyAlgoConfig` to `prime_rl.configs.algorithm` and its discriminated union, then register `"my_algo": MyAlgorithm` in `ALGORITHM_CLASSES`. Pick the hook by *when* your signal is ready: `score_rollout` for per-arrival credit or credit that needs a model call (it's `async`), `score_group` for group-relative credit (GRPO/MaxRL). `assign_advantages` takes a scalar (broadcast over the rollout's trainable tokens — the common case) or a full-length per-token list aligned to the concatenated sample token_ids (process rewards, step-level credit; `0.0` off-mask).
 
-Each per-token list must match the rollout's completion-token count exactly — validated loudly when the view writes it. Advantage-based filters and metrics derive from the streams (the zero-advantage filter checks for all-zero streams; logged distributions use per-rollout means). Signals that depend on the live policy's weights (like OPD's reverse KL) cannot be precomputed here; those are reference-scoring algorithms, evaluated in the trainer.
+Each per-token list must match the rollout's completion-token count exactly — validated loudly when the view writes it. The zero-advantage drop and metrics derive from the streams (the drop checks for all-zero streams; logged distributions use per-rollout means). Signals that depend on the live policy's weights (like OPD's reverse KL) cannot be precomputed here; those are reference-scoring algorithms, evaluated in the trainer.
 
 ### Reference Scoring
 
@@ -454,30 +454,21 @@ type = "opsd"
 demo_key = "demonstration"
 ```
 
-Scoring runs at arrival, *before* the pre-batch filters, so a rollout that is later filtered still cost its reference compute — accepted for the simpler one-rollout-at-a-time shape (advantage-based filters never fire for opd/opsd anyway, since neither assigns an advantage).
+Scoring runs at arrival, *before* the zero-advantage drop, so a rollout that is later dropped still cost its reference compute — accepted for the simpler one-rollout-at-a-time shape (the drop never fires for opd/opsd anyway, since neither assigns an advantage).
 
 ## Filters
 
-Filters drop rollouts between scoring and training. Built-ins (composable):
+Between scoring and training the sink runs three hardcoded checks on every trainable rollout:
 
-| Filter | Effect |
+| Check | Effect |
 |---|---|
-| `gibberish` | Drops rollouts whose mean log-prob fall below a threshold — usually a sign of degenerate output. |
-| `repetition` | Drops rollouts with high n-gram repetition. |
-| `zero_advantage` | Drops rollouts whose advantage is zero, so the trainer doesn't waste tokens on them. |
+| `gibberish` | Detects rare tokens generated at high entropy — usually a sign of degenerate output. Monitor-only: tracked in metrics (`gibberish/mean`, next to the other per-agent trace verdicts), never dropped. |
+| `repetition` | Detects long high-confidence loops. Monitor-only: tracked in metrics (`repetition/mean`), never dropped. |
+| `zero_advantage` | A rollout whose advantage stream is all zero (its whole group earned the same reward) carries no learning signal — dropped before it consumes batch budget, so the trainer never wastes tokens on it. |
 
-The default `[orchestrator]` config registers all three in both filter slots: `post_batch_filters` enforce by default (flagged rollouts are recorded but not shipped to the trainer), while `pre_batch_filters` run in monitor mode (`enforce = false`); flip `enforce = true` there to drop matching rollouts before they consume a slot in the batch. Setting a slot replaces its defaults wholesale:
+Zero-advantage rollouts are exempt when the env's algorithm declares `trains_on_zero_advantage` (echo: the `ce` component trains observation tokens regardless of credit); algorithms that assign no advantage at all (opd/opsd) never match. `orchestrator.count_zero_advantage_in_batch = true` makes dropped rollouts still count toward `batch_size` — a fixed sampling budget per step, at the cost of a variable number of trained-on samples.
 
-```toml
-[[orchestrator.post_batch_filters]]
-type = "zero_advantage"
-
-[[orchestrator.post_batch_filters]]
-type = "repetition"
-threshold = 0.4
-```
-
-Filtered rollouts still appear in W&B distributions, just not in the trainer batch — useful for spotting whether filtering is doing its job.
+Dropped rollouts still appear in W&B distributions and metrics (`is_filtered`, `zero_advantage`), just not in the trainer batch.
 
 ## Multi-Turn Trajectories
 

docs/overview.md

@@ -40,6 +40,6 @@ The `rl` entrypoint reads `examples/basic/reverse-text/rl.toml`, splits it into
 - **[Training](training.md)** — Launch and observe RL and SFT runs.
 - **[Inference](inference.md)** — vLLM-backed server (or fleet) holding the current policy.
 - **[Scaling](scaling.md)** — Single-GPU through multi-node clusters via FSDP / EP / CP and SLURM.
-- **[Algorithms](algorithms.md)** — Async semantics, loss / advantage / filter plugins, trajectory merging.
+- **[Algorithms](algorithms.md)** — Async semantics, loss / advantage plugins, rollout checks, trajectory merging.
 - **[Advanced](advanced.md)** — Custom modeling, multimodal, LoRA, P/D inference.
 - **[Development](development.md)** — Test suite, pre-commit hooks, adding a new model.

docs/training.md

@@ -57,7 +57,8 @@ A condensed view of the knobs you'll most often tune. For trainer-side paralleli
 
 | Knob | What it does |
 |---|---|
-| `orchestrator.batch_size` | Tasks per trainer step. |
+| `orchestrator.batch_size` | Rollouts to train on per step. |
+| `orchestrator.max_inflight_episodes` | Concurrent episodes kept in-flight. Defaults to `batch_size`; raise above it to oversample ahead of the next batch. |
 | `orchestrator.group_size` | Rollouts generated per task. |
 | `orchestrator.max_off_policy_steps` | How many distinct policies may have contributed to one rollout before it's discarded (default 8). The main off-policy dial on long agentic rollouts — bump for throughput, lower for tighter on-policyness. Watch `errored_rollouts` and `mismatch_kl/all/mean` when tuning. |
 | `[orchestrator.algo]` | Training algorithm — its `type` names it (`grpo` default, `max_rl`, `rae`, `hierarchical_grpo`, `opd`, `opsd`, `sft`, `echo`). See [Algorithms](#algorithms). |

examples/advanced/glm-5.2/swe.toml

@@ -66,7 +66,7 @@ weight_decay = 0.1
 [orchestrator]
 batch_size = 4096
 group_size = 16
-oversampling_factor = 3
+max_inflight_episodes = 12_288
 max_off_policy_steps = 16
 
 [orchestrator.model]
@@ -88,10 +88,6 @@ id = "bash"
 type = "prime"
 labels = ["glm5-pd-disag", "swe-bench-verified"]
 
-[[orchestrator.post_batch_filters]]
-type = "gibberish"
-enforce = true
-
 [inference]
 # we need <0.85 bc glm5 layers are too large for 0.85
 use_deep_gemm = true

examples/advanced/intellect-3.1/rl.toml

@@ -45,7 +45,7 @@ weight_decay = 0.01
 
 [orchestrator]
 batch_size = 2048
-oversampling_factor = 2
+max_inflight_episodes = 4096
 
 [[orchestrator.train.source]]
 name = "swe"
@@ -119,10 +119,6 @@ id = "null"
 [orchestrator.train.source.env.agent.runtime]
 type = "subprocess"
 
-[[orchestrator.pre_batch_filters]]
-type = "zero_advantage"
-enforce = true
-
 [orchestrator.eval]
 interval = 25
 

examples/advanced/minimax-m2.5/swe.toml

@@ -45,7 +45,7 @@ weight_decay = 0.01
 
 [orchestrator]
 batch_size = 2048
-oversampling_factor = 2
+max_inflight_episodes = 4096
 max_off_policy_steps = 16
 
 [[orchestrator.train.source]]

examples/advanced/qwen3-30b-a3b/math.toml

@@ -42,7 +42,7 @@ lr = 1e-6
 
 [orchestrator]
 batch_size = 512
-oversampling_factor = 2
+max_inflight_episodes = 1024
 max_off_policy_steps = 8
 
 [orchestrator.train.sampling]

examples/advanced/qwen3-30b-a3b/swe.toml

@@ -43,7 +43,7 @@ lr = 1e-6
 
 [orchestrator]
 batch_size = 512
-oversampling_factor = 2
+max_inflight_episodes = 1024
 max_off_policy_steps = 16
 
 [[orchestrator.train.source]]

examples/basic/wiki-search/rl.toml

@@ -32,7 +32,7 @@ target_modules = [
 [orchestrator]
 batch_size = 512
 group_size = 16
-oversampling_factor = 2.0
+max_inflight_episodes = 1024
 
 [orchestrator.model.lora]
 name = "qwen3-4b-wiki-search"
@@ -52,10 +52,6 @@ id = "null"
 [orchestrator.train.source.env.agent.runtime]
 type = "subprocess"
 
-[[orchestrator.pre_batch_filters]]
-type = "zero_advantage"
-enforce = true
-
 [ckpt] # Checkpoint at the end of training
 
 [inference.vllm]