From db55463505e91942e9a2d40f96a63da9444c04e7 Mon Sep 17 00:00:00 2001 From: hallerite Date: Fri, 24 Jul 2026 14:07:07 +0000 Subject: [PATCH 01/10] chore: bump verifiers to latest main; adopt runtime-on-agent, Reward records, textarena player MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deps/verifiers b13ba60da → 295a06505 (+12: interleaving agents + kuhn-poker-v1, runtime moves from the harness onto the agent (#2106), trace rewards become Reward{score, weight} records (#2119), wordle-v1 rides the rewritten TextArenaEnv, pluggable harness skills, agentic-judge grading knobs). Adoption: - every `env..harness = { ..., runtime = ... }` config line splits into `harness = { ... }` + `env..runtime = ...` (configs/, examples/, docs/, skills/) - wordle configs address the textarena seat as `env.player` - reward metrics read the weighted `Reward.value` (CustomMetrics gains a value extractor); unit-test trace builders record `vf.Reward(score=...)` - uv.lock re-locked for verifiers 0.2.2.dev29 Co-Authored-By: Claude Fable 5 --- configs/basic/alphabet-sort/rl.toml | 3 ++- configs/basic/hendrycks-sanity/rl.toml | 3 ++- configs/basic/reverse-text/rl.toml | 3 ++- configs/basic/wiki-search/rl.toml | 3 ++- configs/basic/wordle/rl.toml | 3 ++- configs/ci/integration/alphabet_sort.toml | 3 ++- .../integration/reverse-text-lora/resume.toml | 3 ++- .../integration/reverse-text-lora/start.toml | 3 ++- .../integration/reverse-text-moe/start.toml | 3 ++- .../reverse-text-multi-run/orchestrator.toml | 3 ++- .../reverse-text-rl-opd/start.toml | 6 ++++-- .../reverse-text-rl-sft/start.toml | 6 ++++-- .../ci/integration/reverse-text/resume.toml | 3 ++- .../ci/integration/reverse-text/start.toml | 3 ++- configs/ci/nightly-fft/alphabet-sort.toml | 3 ++- configs/ci/nightly-fft/hendrycks-sanity.toml | 6 ++++-- configs/ci/nightly-fft/reverse-text.toml | 3 ++- configs/ci/nightly-fft/wiki-search.toml | 3 ++- configs/ci/nightly-fft/wordle.toml | 3 ++- .../ci/nightly/multimodal_color_codeword.toml | 3 ++- configs/debug/algo/echo.toml | 3 ++- configs/debug/algo/grpo.toml | 6 ++++-- configs/debug/algo/max_rl.toml | 6 ++++-- configs/debug/algo/mixed_grpo_opd.toml | 9 +++++--- configs/debug/algo/opd.toml | 6 ++++-- configs/debug/algo/opd_lora.toml | 6 ++++-- configs/debug/algo/self_distill.toml | 6 ++++-- configs/debug/algo/sft_distill.toml | 6 ++++-- configs/debug/algo/sft_distill_lora.toml | 6 ++++-- deps/verifiers | 2 +- docs/algorithms.md | 6 ++++-- docs/configuration.md | 9 +++++--- examples/advanced/glm-4.5-air/search.toml | 9 +++++--- examples/advanced/glm-4.5-air/swe.toml | 6 ++++-- examples/advanced/glm-4.5-air/terminal.toml | 9 +++++--- examples/advanced/glm-5.2/swe-llmd.toml | 3 ++- examples/advanced/glm-5.2/swe.toml | 3 ++- examples/advanced/intellect-3.1/rl.toml | 21 ++++++++++++------- examples/advanced/minimax-m2.5/swe.toml | 6 ++++-- examples/advanced/nemotron-3-super/swe.toml | 6 ++++-- examples/advanced/qwen3-30b-a3b/math.toml | 8 ++++--- examples/advanced/qwen3-30b-a3b/swe.toml | 8 ++++--- examples/advanced/qwen3-30b-a3b/tool.toml | 3 ++- examples/basic/alphabet-sort/rl.toml | 3 ++- examples/basic/hendrycks-sanity/rl.toml | 6 ++++-- examples/basic/reverse-text/rl.toml | 3 ++- examples/basic/wiki-search/README.md | 3 ++- examples/basic/wiki-search/rl.toml | 3 ++- examples/basic/wordle/rl.toml | 3 ++- skills/configs/SKILL.md | 3 ++- src/prime_rl/orchestrator/metrics.py | 17 +++++++++------ tests/unit/orchestrator/test_advantage.py | 2 +- tests/unit/orchestrator/test_algorithms.py | 4 ++-- tests/unit/orchestrator/test_filters.py | 6 ++++-- tests/unit/orchestrator/test_metrics.py | 5 +++-- uv.lock | 21 +++++++++++++++---- 56 files changed, 201 insertions(+), 100 deletions(-) diff --git a/configs/basic/alphabet-sort/rl.toml b/configs/basic/alphabet-sort/rl.toml index 8363e1e3ff2..4b91f67a192 100644 --- a/configs/basic/alphabet-sort/rl.toml +++ b/configs/basic/alphabet-sort/rl.toml @@ -35,7 +35,8 @@ max_completion_tokens = 768 [[orchestrator.train.env]] name = "alphabet-sort" env.taskset = { id = "alphabet-sort-v1", min_turns = 3, max_turns = 5, task = { power_per_turn = false } } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [ckpt] # Checkpoint at the end of training diff --git a/configs/basic/hendrycks-sanity/rl.toml b/configs/basic/hendrycks-sanity/rl.toml index 0512feada3c..422720c253b 100644 --- a/configs/basic/hendrycks-sanity/rl.toml +++ b/configs/basic/hendrycks-sanity/rl.toml @@ -22,7 +22,8 @@ seq_len = 8192 [[orchestrator.train.env]] name = "hendrycks-math" env.taskset = { id = "math-env-v1", dataset_name = "mikasenghaas/Sanity-Test-R1D-1.5B", dataset_subset = "default" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.model] seq_len = 16384 diff --git a/configs/basic/reverse-text/rl.toml b/configs/basic/reverse-text/rl.toml index 5d80c55ac21..3f7ee602c28 100644 --- a/configs/basic/reverse-text/rl.toml +++ b/configs/basic/reverse-text/rl.toml @@ -25,7 +25,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.optim] lr = 3e-6 diff --git a/configs/basic/wiki-search/rl.toml b/configs/basic/wiki-search/rl.toml index f69278c95c1..c1ec50655f0 100644 --- a/configs/basic/wiki-search/rl.toml +++ b/configs/basic/wiki-search/rl.toml @@ -43,7 +43,8 @@ max_completion_tokens = 512 [[orchestrator.train.env]] name = "wiki-search" env.taskset = { id = "wiki-search-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [ckpt] # Checkpoint at the end of training diff --git a/configs/basic/wordle/rl.toml b/configs/basic/wordle/rl.toml index aec07641e1d..5d46d655eb8 100644 --- a/configs/basic/wordle/rl.toml +++ b/configs/basic/wordle/rl.toml @@ -24,7 +24,8 @@ group_size = 8 [[orchestrator.train.env]] name = "wordle" env.taskset = { id = "wordle-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.player.harness = { id = "null" } +env.player.runtime = { type = "subprocess" } [orchestrator.train.sampling] max_completion_tokens = 1024 diff --git a/configs/ci/integration/alphabet_sort.toml b/configs/ci/integration/alphabet_sort.toml index 1c13176580d..f63f8636b1d 100644 --- a/configs/ci/integration/alphabet_sort.toml +++ b/configs/ci/integration/alphabet_sort.toml @@ -25,7 +25,8 @@ max_completion_tokens = 384 [[orchestrator.train.env]] name = "alphabet-sort" env.taskset = { id = "alphabet-sort-v1", min_turns = 2, max_turns = 2, min_names_per_turn = 1, max_names_per_turn = 3, task = { similarity_power = 4, power_per_turn = false } } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [inference] diff --git a/configs/ci/integration/reverse-text-lora/resume.toml b/configs/ci/integration/reverse-text-lora/resume.toml index 7c655028327..372cfc4c9ad 100644 --- a/configs/ci/integration/reverse-text-lora/resume.toml +++ b/configs/ci/integration/reverse-text-lora/resume.toml @@ -29,7 +29,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [inference] enable_lora = true diff --git a/configs/ci/integration/reverse-text-lora/start.toml b/configs/ci/integration/reverse-text-lora/start.toml index f10981bee2f..e3b867d7dd6 100644 --- a/configs/ci/integration/reverse-text-lora/start.toml +++ b/configs/ci/integration/reverse-text-lora/start.toml @@ -28,7 +28,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [inference] enable_lora = true diff --git a/configs/ci/integration/reverse-text-moe/start.toml b/configs/ci/integration/reverse-text-moe/start.toml index c695e80da26..e4a0093d604 100644 --- a/configs/ci/integration/reverse-text-moe/start.toml +++ b/configs/ci/integration/reverse-text-moe/start.toml @@ -28,7 +28,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [inference] gpu_memory_utilization = 0.7 diff --git a/configs/ci/integration/reverse-text-multi-run/orchestrator.toml b/configs/ci/integration/reverse-text-multi-run/orchestrator.toml index f60ffc5c2ff..dab5fe8cdd7 100644 --- a/configs/ci/integration/reverse-text-multi-run/orchestrator.toml +++ b/configs/ci/integration/reverse-text-multi-run/orchestrator.toml @@ -17,7 +17,8 @@ max_completion_tokens = 128 [[train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [ckpt] interval = 10 diff --git a/configs/ci/integration/reverse-text-rl-opd/start.toml b/configs/ci/integration/reverse-text-rl-opd/start.toml index 0355275804c..20699400084 100644 --- a/configs/ci/integration/reverse-text-rl-opd/start.toml +++ b/configs/ci/integration/reverse-text-rl-opd/start.toml @@ -40,7 +40,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 1 @@ -52,7 +53,8 @@ max_completion_tokens = 128 [[orchestrator.eval.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.optim] lr = 3e-6 diff --git a/configs/ci/integration/reverse-text-rl-sft/start.toml b/configs/ci/integration/reverse-text-rl-sft/start.toml index 8b46a4b5de4..c8b3d08a8bf 100644 --- a/configs/ci/integration/reverse-text-rl-sft/start.toml +++ b/configs/ci/integration/reverse-text-rl-sft/start.toml @@ -46,7 +46,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 1 @@ -58,7 +59,8 @@ max_completion_tokens = 128 [[orchestrator.eval.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.optim] lr = 3e-6 diff --git a/configs/ci/integration/reverse-text/resume.toml b/configs/ci/integration/reverse-text/resume.toml index e1163b5695d..06fab44ac54 100644 --- a/configs/ci/integration/reverse-text/resume.toml +++ b/configs/ci/integration/reverse-text/resume.toml @@ -26,7 +26,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [inference] diff --git a/configs/ci/integration/reverse-text/start.toml b/configs/ci/integration/reverse-text/start.toml index c2aea7845b6..11facde4c84 100644 --- a/configs/ci/integration/reverse-text/start.toml +++ b/configs/ci/integration/reverse-text/start.toml @@ -25,7 +25,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [inference] diff --git a/configs/ci/nightly-fft/alphabet-sort.toml b/configs/ci/nightly-fft/alphabet-sort.toml index 8540d290571..72e4ef453a9 100644 --- a/configs/ci/nightly-fft/alphabet-sort.toml +++ b/configs/ci/nightly-fft/alphabet-sort.toml @@ -21,7 +21,8 @@ group_size = 16 [[orchestrator.train.env]] name = "alphabet-sort" env.taskset = { id = "alphabet-sort-v1", min_turns = 3, max_turns = 5, task = { power_per_turn = false } } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.train.sampling] max_completion_tokens = 768 diff --git a/configs/ci/nightly-fft/hendrycks-sanity.toml b/configs/ci/nightly-fft/hendrycks-sanity.toml index a2e8ca5fb86..575ee690edc 100644 --- a/configs/ci/nightly-fft/hendrycks-sanity.toml +++ b/configs/ci/nightly-fft/hendrycks-sanity.toml @@ -21,7 +21,8 @@ seq_len = 8192 [[orchestrator.train.env]] name = "hendrycks-math" env.taskset = { id = "math-env-v1", dataset_name = "mikasenghaas/Sanity-Test-R1D-1.5B", dataset_subset = "default" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 50 @@ -29,7 +30,8 @@ interval = 50 [[orchestrator.eval.env]] name = "aime2024" env.taskset = { id = "aime24-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } group_size = 32 [trainer] diff --git a/configs/ci/nightly-fft/reverse-text.toml b/configs/ci/nightly-fft/reverse-text.toml index 36a1878eb64..73d95d0050f 100644 --- a/configs/ci/nightly-fft/reverse-text.toml +++ b/configs/ci/nightly-fft/reverse-text.toml @@ -21,7 +21,8 @@ group_size = 16 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.train.sampling] max_completion_tokens = 128 diff --git a/configs/ci/nightly-fft/wiki-search.toml b/configs/ci/nightly-fft/wiki-search.toml index 17d745ce995..8b90fdae61d 100644 --- a/configs/ci/nightly-fft/wiki-search.toml +++ b/configs/ci/nightly-fft/wiki-search.toml @@ -26,7 +26,8 @@ enforce = true [[orchestrator.train.env]] name = "wiki-search" env.taskset = { id = "wiki-search-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.train.sampling] max_completion_tokens = 512 diff --git a/configs/ci/nightly-fft/wordle.toml b/configs/ci/nightly-fft/wordle.toml index 694986f2cc0..ad10225aebd 100644 --- a/configs/ci/nightly-fft/wordle.toml +++ b/configs/ci/nightly-fft/wordle.toml @@ -21,7 +21,8 @@ group_size = 16 [[orchestrator.train.env]] name = "wordle" env.taskset = { id = "wordle-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.player.harness = { id = "null" } +env.player.runtime = { type = "subprocess" } [orchestrator.train.sampling] max_completion_tokens = 1024 diff --git a/configs/ci/nightly/multimodal_color_codeword.toml b/configs/ci/nightly/multimodal_color_codeword.toml index 4c8d980cb4c..5ce49719985 100644 --- a/configs/ci/nightly/multimodal_color_codeword.toml +++ b/configs/ci/nightly/multimodal_color_codeword.toml @@ -23,7 +23,8 @@ max_completion_tokens = 64 [[orchestrator.train.env]] name = "color-codeword" env.taskset = { id = "color-codeword-v1", images_per_turn = 1 } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer] diff --git a/configs/debug/algo/echo.toml b/configs/debug/algo/echo.toml index 88ca238fdb2..0805249c2b9 100644 --- a/configs/debug/algo/echo.toml +++ b/configs/debug/algo/echo.toml @@ -27,7 +27,8 @@ alpha = 0.1 [[orchestrator.train.env]] name = "alphabet-sort" env.taskset = { id = "alphabet-sort-v1", min_turns = 3, max_turns = 5, task = { power_per_turn = false } } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.train.sampling] max_completion_tokens = 512 diff --git a/configs/debug/algo/grpo.toml b/configs/debug/algo/grpo.toml index ea1fa4b999a..25da69ec8d7 100644 --- a/configs/debug/algo/grpo.toml +++ b/configs/debug/algo/grpo.toml @@ -24,7 +24,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 1 @@ -36,7 +37,8 @@ max_completion_tokens = 128 [[orchestrator.eval.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.optim] lr = 3e-6 diff --git a/configs/debug/algo/max_rl.toml b/configs/debug/algo/max_rl.toml index 9c416aab2d0..620fbd2b986 100644 --- a/configs/debug/algo/max_rl.toml +++ b/configs/debug/algo/max_rl.toml @@ -24,7 +24,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 1 @@ -36,7 +37,8 @@ max_completion_tokens = 128 [[orchestrator.eval.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.optim] lr = 3e-6 diff --git a/configs/debug/algo/mixed_grpo_opd.toml b/configs/debug/algo/mixed_grpo_opd.toml index 5e55450e601..a4a24ec4b43 100644 --- a/configs/debug/algo/mixed_grpo_opd.toml +++ b/configs/debug/algo/mixed_grpo_opd.toml @@ -34,12 +34,14 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text-grpo" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [[orchestrator.train.env]] name = "reverse-text-opd" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.train.env.algo] type = "opd" @@ -58,7 +60,8 @@ max_completion_tokens = 128 [[orchestrator.eval.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.optim] lr = 3e-6 diff --git a/configs/debug/algo/opd.toml b/configs/debug/algo/opd.toml index 53e9eac252e..13d2950c731 100644 --- a/configs/debug/algo/opd.toml +++ b/configs/debug/algo/opd.toml @@ -36,7 +36,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 1 @@ -48,7 +49,8 @@ max_completion_tokens = 128 [[orchestrator.eval.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.optim] lr = 3e-6 diff --git a/configs/debug/algo/opd_lora.toml b/configs/debug/algo/opd_lora.toml index 948bb5124f5..a6f0eb95509 100644 --- a/configs/debug/algo/opd_lora.toml +++ b/configs/debug/algo/opd_lora.toml @@ -36,7 +36,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 1 @@ -48,7 +49,8 @@ max_completion_tokens = 128 [[orchestrator.eval.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.optim] lr = 1e-4 diff --git a/configs/debug/algo/self_distill.toml b/configs/debug/algo/self_distill.toml index 25344d1f005..93959f7b222 100644 --- a/configs/debug/algo/self_distill.toml +++ b/configs/debug/algo/self_distill.toml @@ -32,7 +32,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 1 @@ -44,7 +45,8 @@ max_completion_tokens = 128 [[orchestrator.eval.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.optim] lr = 3e-6 diff --git a/configs/debug/algo/sft_distill.toml b/configs/debug/algo/sft_distill.toml index c34dac0d1db..dbc0986f2c1 100644 --- a/configs/debug/algo/sft_distill.toml +++ b/configs/debug/algo/sft_distill.toml @@ -41,7 +41,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 1 @@ -53,7 +54,8 @@ max_completion_tokens = 128 [[orchestrator.eval.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.optim] lr = 3e-6 diff --git a/configs/debug/algo/sft_distill_lora.toml b/configs/debug/algo/sft_distill_lora.toml index ad05f653a41..51f1034c6a0 100644 --- a/configs/debug/algo/sft_distill_lora.toml +++ b/configs/debug/algo/sft_distill_lora.toml @@ -41,7 +41,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 1 @@ -53,7 +54,8 @@ max_completion_tokens = 128 [[orchestrator.eval.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.optim] lr = 1e-4 diff --git a/deps/verifiers b/deps/verifiers index b13ba60da63..295a0650585 160000 --- a/deps/verifiers +++ b/deps/verifiers @@ -1 +1 @@ -Subproject commit b13ba60da63cea91389e7575766b7270d0d11fc5 +Subproject commit 295a0650585af8754eaf98c6fa9071314b5a531c diff --git a/docs/algorithms.md b/docs/algorithms.md index eb114e58caf..8088d28516f 100644 --- a/docs/algorithms.md +++ b/docs/algorithms.md @@ -119,13 +119,15 @@ type = "grpo" [[orchestrator.train.env]] name = "math" taskset = { id = "math-v1" } -harness = { id = "null", runtime = { type = "subprocess" } } +harness = { id = "null" } +runtime = { type = "subprocess" } # inherits the top-level grpo [[orchestrator.train.env]] name = "terminal" taskset = { id = "terminal-v1" } -harness = { id = "bash", runtime = { type = "subprocess" } } +harness = { id = "bash" } +runtime = { type = "subprocess" } algo = { type = "echo" } # this env runs its own algorithm ``` diff --git a/docs/configuration.md b/docs/configuration.md index e624f471aaa..4b0474edeba 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -150,19 +150,22 @@ Training environments are an array of tables — set one per env, optionally wit [[orchestrator.train.env]] name = "gsm8k" env.taskset = { id = "gsm8k-v1", split = "train" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } ratio = 3 # 75% of batches [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } ratio = 1 # default — 25% of batches [[orchestrator.eval.env]] name = "gsm8k-eval" env.taskset = { id = "gsm8k-v1", split = "test" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } ``` `ratio` defaults to `1` (equal weight per env); values are relative weights normalized to probabilities across envs. diff --git a/examples/advanced/glm-4.5-air/search.toml b/examples/advanced/glm-4.5-air/search.toml index cc7c4290f65..e2d3dfc997c 100644 --- a/examples/advanced/glm-4.5-air/search.toml +++ b/examples/advanced/glm-4.5-air/search.toml @@ -95,7 +95,8 @@ num_turns_weight = 0.0 # the plugged BC-style prompt is replaced by the reference judge's stock prompt). [[orchestrator.train.env]] name = "openseeker" -env.agent.harness = { id = "rlm", skills = ["search"], forward_env = ["SERPER_API_KEY"], summarize_at_tokens = [65536, 131072], runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-search"] } } +env.agent.harness = { id = "rlm", skills = ["search"], forward_env = ["SERPER_API_KEY"], summarize_at_tokens = [65536, 131072] } +env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-search"] } [orchestrator.train.env.env.taskset] id = "openseeker-v1" @@ -117,7 +118,8 @@ choices = ["A", "B"] [[orchestrator.train.env]] name = "redsearcher" -env.agent.harness = { id = "rlm", skills = ["search"], forward_env = ["SERPER_API_KEY"], summarize_at_tokens = [65536, 131072], runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-search"] } } +env.agent.harness = { id = "rlm", skills = ["search"], forward_env = ["SERPER_API_KEY"], summarize_at_tokens = [65536, 131072] } +env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-search"] } [orchestrator.train.env.env.taskset] id = "redsearcher-v1" # optional `difficulty` filter (easy/medium/hard) @@ -145,7 +147,8 @@ interval = 20 [[orchestrator.eval.env]] name = "browsecomp" num_examples = 500 # full set is 1266; cap eval at 500 -env.agent.harness = { id = "rlm", skills = ["search"], forward_env = ["SERPER_API_KEY"], summarize_at_tokens = 98304, runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-search"] } } +env.agent.harness = { id = "rlm", skills = ["search"], forward_env = ["SERPER_API_KEY"], summarize_at_tokens = 98304 } +env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-search"] } env.agent.timeout = { rollout = 3600 } [orchestrator.eval.env.env.taskset] diff --git a/examples/advanced/glm-4.5-air/swe.toml b/examples/advanced/glm-4.5-air/swe.toml index c63d8eb4362..b9de4512cce 100644 --- a/examples/advanced/glm-4.5-air/swe.toml +++ b/examples/advanced/glm-4.5-air/swe.toml @@ -100,7 +100,8 @@ thinking_retention = "all" [[orchestrator.train.env]] name = "scaleswe" env.taskset = { id = "scaleswe-v1" } -env.agent.harness = { id = "rlm", summarize_at_tokens = [65536, 131072], runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-swe"] } } +env.agent.harness = { id = "rlm", summarize_at_tokens = [65536, 131072] } +env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-swe"] } [orchestrator.prime_monitor] @@ -112,7 +113,8 @@ interval = 20 [[orchestrator.eval.env]] name = "swebench-verified" env.taskset = { id = "swebench-verified-v1" } -env.agent.harness = { id = "rlm", summarize_at_tokens = 98304, runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-swe"] } } +env.agent.harness = { id = "rlm", summarize_at_tokens = 98304 } +env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-swe"] } env.agent.timeout = { rollout = 3600 } # --- Inference --- diff --git a/examples/advanced/glm-4.5-air/terminal.toml b/examples/advanced/glm-4.5-air/terminal.toml index 36a888474d7..7dfd984e1aa 100644 --- a/examples/advanced/glm-4.5-air/terminal.toml +++ b/examples/advanced/glm-4.5-air/terminal.toml @@ -93,7 +93,8 @@ num_turns_weight = 0.1 [[orchestrator.train.env]] name = "tmax" -env.agent.harness = { id = "rlm", summarize_at_tokens = [65536, 131072], runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-terminal"] } } +env.agent.harness = { id = "rlm", summarize_at_tokens = [65536, 131072] } +env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-terminal"] } [orchestrator.train.env.env.taskset] id = "tmax-v1" @@ -120,7 +121,8 @@ interval = 20 [[orchestrator.eval.env]] name = "swebench-verified" -env.agent.harness = { id = "rlm", summarize_at_tokens = 98304, runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-terminal"] } } +env.agent.harness = { id = "rlm", summarize_at_tokens = 98304 } +env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-terminal"] } env.agent.timeout = { rollout = 3600 } [orchestrator.eval.env.env.taskset] @@ -143,7 +145,8 @@ id = "swebench-verified-v1" [[orchestrator.eval.env]] name = "terminal-bench-2" group_size = 4 # avg@4 -env.agent.harness = { id = "rlm", summarize_at_tokens = 98304, runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-terminal"] } } +env.agent.harness = { id = "rlm", summarize_at_tokens = 98304 } +env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-terminal"] } env.agent.timeout = { rollout = 3600 } [orchestrator.eval.env.env.taskset] diff --git a/examples/advanced/glm-5.2/swe-llmd.toml b/examples/advanced/glm-5.2/swe-llmd.toml index 8e544235fb7..4fdb589961b 100644 --- a/examples/advanced/glm-5.2/swe-llmd.toml +++ b/examples/advanced/glm-5.2/swe-llmd.toml @@ -99,7 +99,8 @@ extra_body = {chat_template_kwargs = {clear_thinking = false}} [[orchestrator.train.env]] name = "r2e" env.taskset = { id = "r2e-gym-v1" } -env.agent.harness = { id = "rlm", runtime = { type = "prime", labels = ["glm5-llmd"] } } +env.agent.harness = { id = "rlm" } +env.agent.runtime = { type = "prime", labels = ["glm5-llmd"] } [inference] enable_expert_parallel = true diff --git a/examples/advanced/glm-5.2/swe.toml b/examples/advanced/glm-5.2/swe.toml index 3e2aa33c52d..99d2b31f70a 100644 --- a/examples/advanced/glm-5.2/swe.toml +++ b/examples/advanced/glm-5.2/swe.toml @@ -74,7 +74,8 @@ interval = 20 [[orchestrator.eval.env]] name = "swe-bench-verified" env.taskset = { id = "swebench-verified-v1" } -env.agent.harness = { id = "bash", runtime = { type = "prime", labels = ["glm5-pd-disag", "swe-bench-verified"] } } +env.agent.harness = { id = "bash" } +env.agent.runtime = { type = "prime", labels = ["glm5-pd-disag", "swe-bench-verified"] } [[orchestrator.post_batch_filters]] type = "gibberish" diff --git a/examples/advanced/intellect-3.1/rl.toml b/examples/advanced/intellect-3.1/rl.toml index 373a4d9dc7b..a85ba04cdc9 100644 --- a/examples/advanced/intellect-3.1/rl.toml +++ b/examples/advanced/intellect-3.1/rl.toml @@ -51,31 +51,36 @@ oversampling_factor = 2 name = "swe" ratio = 0.3 env.taskset = { id = "r2e-gym-v1" } -env.agent.harness = { id = "bash", runtime = { type = "prime", labels = ["intellect-3.1", "swe"] } } +env.agent.harness = { id = "bash" } +env.agent.runtime = { type = "prime", labels = ["intellect-3.1", "swe"] } [[orchestrator.train.env]] name = "deepdive" ratio = 0.2 env.taskset = { id = "deepdive-v1" } -env.agent.harness = { id = "rlm", skills = ["search"], forward_env = ["SERPER_API_KEY"], runtime = { type = "prime", labels = ["intellect-3.1", "deepdive"] } } +env.agent.harness = { id = "rlm", skills = ["search"], forward_env = ["SERPER_API_KEY"] } +env.agent.runtime = { type = "prime", labels = ["intellect-3.1", "deepdive"] } [[orchestrator.train.env]] name = "math" ratio = 0.3 env.taskset = { id = "i3_math_v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [[orchestrator.train.env]] name = "logic" ratio = 0.2 env.taskset = { id = "i3_logic_v1", filter = { max = 0.874 } } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [[orchestrator.train.env]] name = "code" ratio = 0.2 env.taskset = { id = "i3_code_v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [[orchestrator.pre_batch_filters]] type = "zero_advantage" @@ -87,12 +92,14 @@ interval = 25 [[orchestrator.eval.env]] name = "swe-bench-verified" env.taskset = { id = "swebench-verified-v1" } -env.agent.harness = { id = "bash", runtime = { type = "prime", labels = ["intellect-3.1", "swe-bench-verified"] } } +env.agent.harness = { id = "bash" } +env.agent.runtime = { type = "prime", labels = ["intellect-3.1", "swe-bench-verified"] } [[orchestrator.eval.env]] name = "aime2025" env.taskset = { id = "aime25-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [inference.parallel] tp = 8 diff --git a/examples/advanced/minimax-m2.5/swe.toml b/examples/advanced/minimax-m2.5/swe.toml index 9f2dd8d7df5..76035905c4d 100644 --- a/examples/advanced/minimax-m2.5/swe.toml +++ b/examples/advanced/minimax-m2.5/swe.toml @@ -52,7 +52,8 @@ max_off_policy_steps = 16 [[orchestrator.train.env]] name = "swe" env.taskset = { id = "r2e-gym-v1" } -env.agent.harness = { id = "bash", runtime = { type = "prime", labels = ["minimax-swe", "swe"] } } +env.agent.harness = { id = "bash" } +env.agent.runtime = { type = "prime", labels = ["minimax-swe", "swe"] } [orchestrator.eval] interval = 25 @@ -60,7 +61,8 @@ interval = 25 [[orchestrator.eval.env]] name = "swe-bench-verified" env.taskset = { id = "swebench-verified-v1" } -env.agent.harness = { id = "bash", runtime = { type = "prime", labels = ["minimax-swe", "swe-bench-verified"] } } +env.agent.harness = { id = "bash" } +env.agent.runtime = { type = "prime", labels = ["minimax-swe", "swe-bench-verified"] } [inference.parallel] tp = 8 diff --git a/examples/advanced/nemotron-3-super/swe.toml b/examples/advanced/nemotron-3-super/swe.toml index 9b625a934fc..139ff77e04e 100644 --- a/examples/advanced/nemotron-3-super/swe.toml +++ b/examples/advanced/nemotron-3-super/swe.toml @@ -84,7 +84,8 @@ truncate_history_thinking = false [[orchestrator.train.env]] name = "scaleswe" env.taskset = { id = "scaleswe-v1" } -env.agent.harness = { id = "rlm", summarize_at_tokens = [65536, 131072], runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["nemotron-3-super-swe"] } } +env.agent.harness = { id = "rlm", summarize_at_tokens = [65536, 131072] } +env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["nemotron-3-super-swe"] } [orchestrator.prime_monitor] @@ -94,7 +95,8 @@ interval = 20 [[orchestrator.eval.env]] name = "swebench-verified" env.taskset = { id = "swebench-verified-v1" } -env.agent.harness = { id = "rlm", summarize_at_tokens = 98304, runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["nemotron-3-super-swe"] } } +env.agent.harness = { id = "rlm", summarize_at_tokens = 98304 } +env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["nemotron-3-super-swe"] } env.agent.timeout = { rollout = 3600 } [inference] diff --git a/examples/advanced/qwen3-30b-a3b/math.toml b/examples/advanced/qwen3-30b-a3b/math.toml index 3fefccc54f0..4b9f326da18 100644 --- a/examples/advanced/qwen3-30b-a3b/math.toml +++ b/examples/advanced/qwen3-30b-a3b/math.toml @@ -51,7 +51,8 @@ max_completion_tokens = 32768 [[orchestrator.train.env]] name = "math" env.taskset = { id = "i3_math_v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 25 @@ -59,7 +60,8 @@ interval = 25 [[orchestrator.eval.env]] name = "aime2025" env.taskset = { id = "aime25-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [inference.parallel] -tp = 8 \ No newline at end of file +tp = 8 diff --git a/examples/advanced/qwen3-30b-a3b/swe.toml b/examples/advanced/qwen3-30b-a3b/swe.toml index 2eff6c9b0c0..06b50eeba90 100644 --- a/examples/advanced/qwen3-30b-a3b/swe.toml +++ b/examples/advanced/qwen3-30b-a3b/swe.toml @@ -49,7 +49,8 @@ max_off_policy_steps = 16 [[orchestrator.train.env]] name = "swe" env.taskset = { id = "r2e-gym-v1" } -env.agent.harness = { id = "bash", runtime = { type = "prime", labels = ["qwen30b-swe", "swe"] } } +env.agent.harness = { id = "bash" } +env.agent.runtime = { type = "prime", labels = ["qwen30b-swe", "swe"] } [orchestrator.eval] interval = 25 @@ -57,7 +58,8 @@ interval = 25 [[orchestrator.eval.env]] name = "swe-bench-verified" env.taskset = { id = "swebench-verified-v1" } -env.agent.harness = { id = "bash", runtime = { type = "prime", labels = ["qwen30b-swe", "swe-bench-verified"] } } +env.agent.harness = { id = "bash" } +env.agent.runtime = { type = "prime", labels = ["qwen30b-swe", "swe-bench-verified"] } [inference.parallel] -tp = 8 \ No newline at end of file +tp = 8 diff --git a/examples/advanced/qwen3-30b-a3b/tool.toml b/examples/advanced/qwen3-30b-a3b/tool.toml index 36243892003..375d66afd66 100644 --- a/examples/advanced/qwen3-30b-a3b/tool.toml +++ b/examples/advanced/qwen3-30b-a3b/tool.toml @@ -37,7 +37,8 @@ max_off_policy_steps = 32 [[orchestrator.train.env]] env.taskset = { id = "general-agent-v1", task = { tools = { colocated = true } }, min_tier = 1 } -env.agent.harness = { id = "null", runtime = { type = "modal" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "modal" } [inference] gpu_memory_utilization = 0.85 diff --git a/examples/basic/alphabet-sort/rl.toml b/examples/basic/alphabet-sort/rl.toml index 114e2b16457..252de1bc937 100644 --- a/examples/basic/alphabet-sort/rl.toml +++ b/examples/basic/alphabet-sort/rl.toml @@ -34,6 +34,7 @@ max_completion_tokens = 768 [[orchestrator.train.env]] name = "alphabet-sort" env.taskset = { id = "alphabet-sort-v1", min_turns = 3, max_turns = 5, task = { power_per_turn = false } } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [inference] # Default inference config diff --git a/examples/basic/hendrycks-sanity/rl.toml b/examples/basic/hendrycks-sanity/rl.toml index e10198a16f8..40812973a1c 100644 --- a/examples/basic/hendrycks-sanity/rl.toml +++ b/examples/basic/hendrycks-sanity/rl.toml @@ -19,7 +19,8 @@ seq_len = 8192 [[orchestrator.train.env]] name = "hendrycks-math" env.taskset = { id = "math-env-v1", dataset_name = "mikasenghaas/Sanity-Test-R1D-1.5B", dataset_subset = "default" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 50 @@ -27,7 +28,8 @@ interval = 50 [[orchestrator.eval.env]] name = "aime2024" env.taskset = { id = "aime24-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } group_size = 32 [trainer.model] diff --git a/examples/basic/reverse-text/rl.toml b/examples/basic/reverse-text/rl.toml index 9cf5d9a49ce..08990f52a48 100644 --- a/examples/basic/reverse-text/rl.toml +++ b/examples/basic/reverse-text/rl.toml @@ -18,7 +18,8 @@ max_completion_tokens = 128 [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [trainer.optim] lr = 3e-6 diff --git a/examples/basic/wiki-search/README.md b/examples/basic/wiki-search/README.md index 3e82065e4f3..a69009448ce 100644 --- a/examples/basic/wiki-search/README.md +++ b/examples/basic/wiki-search/README.md @@ -136,7 +136,8 @@ The V1 taskset fixes the question bank and searchable corpus. You can replace it [[orchestrator.train.env]] name = "wiki-search" taskset = { id = "wiki-search-v1", task = { judges = [{ id = "reference", model = "openai/gpt-5.4-nano" }] } } -harness = { id = "null", runtime = { type = "subprocess" } } +harness = { id = "null" } +runtime = { type = "subprocess" } ``` ## Notes diff --git a/examples/basic/wiki-search/rl.toml b/examples/basic/wiki-search/rl.toml index eb757e6360a..f88913036cc 100644 --- a/examples/basic/wiki-search/rl.toml +++ b/examples/basic/wiki-search/rl.toml @@ -47,7 +47,8 @@ enforce = true [[orchestrator.train.env]] name = "wiki-search" env.taskset = { id = "wiki-search-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [ckpt] # Checkpoint at the end of training diff --git a/examples/basic/wordle/rl.toml b/examples/basic/wordle/rl.toml index 7db5537abec..5e40e826293 100644 --- a/examples/basic/wordle/rl.toml +++ b/examples/basic/wordle/rl.toml @@ -21,7 +21,8 @@ group_size = 16 [[orchestrator.train.env]] name = "wordle" env.taskset = { id = "wordle-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.player.harness = { id = "null" } +env.player.runtime = { type = "subprocess" } [orchestrator.train.sampling] max_completion_tokens = 1024 diff --git a/skills/configs/SKILL.md b/skills/configs/SKILL.md index 1aa0d006899..81535642770 100644 --- a/skills/configs/SKILL.md +++ b/skills/configs/SKILL.md @@ -44,7 +44,8 @@ Incompatible combinations (e.g. CP requires flash attention) must raise in a `mo [[orchestrator.train.env]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } -env.agent.harness = { id = "null", runtime = { type = "subprocess" } } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } ``` CLI: `--orchestrator.train.env.0.env.taskset.id reverse-text-v1`. diff --git a/src/prime_rl/orchestrator/metrics.py b/src/prime_rl/orchestrator/metrics.py index 859eec29d5b..9b39345d0b7 100644 --- a/src/prime_rl/orchestrator/metrics.py +++ b/src/prime_rl/orchestrator/metrics.py @@ -12,7 +12,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Iterator, Literal +from typing import TYPE_CHECKING, Any, Callable, Iterator, Literal from prime_rl.orchestrator.utils import compute_pass_metrics @@ -134,16 +134,20 @@ def stats(self) -> dict[str, Stat]: class CustomMetrics(StatGroup): """Per-key ``Stat``s over a dynamic per-rollout dict attribute (env ``@metric``s or reward - components), each averaged over the rollouts that report the key.""" + components), each averaged over the rollouts that report the key. ``value`` extracts the + float from each entry (rewards are ``vf.Reward`` records; metrics are plain floats).""" - def __init__(self, rollouts: list[Rollout], attr: str) -> None: + def __init__(self, rollouts: list[Rollout], attr: str, value: Callable[[Any], float] = float) -> None: super().__init__(rollouts) self.attr = attr + self.value = value def stats(self) -> dict[str, Stat]: names = sorted({name for r in self.rollouts for name in getattr(r, self.attr)}) return { - name: Stat([getattr(r, self.attr)[name] for r in self.rollouts if name in getattr(r, self.attr)]) + name: Stat( + [self.value(getattr(r, self.attr)[name]) for r in self.rollouts if name in getattr(r, self.attr)] + ) for name in names } @@ -188,8 +192,9 @@ def metrics(self) -> CustomMetrics: @property def rewards(self) -> CustomMetrics: - """Per-component reward breakdown, keyed by name (summed into the scalar ``reward``).""" - return CustomMetrics(self.rollouts, "rewards") + """Per-component reward breakdown, keyed by name (each entry's weighted ``value``, + summed into the scalar ``reward``).""" + return CustomMetrics(self.rollouts, "rewards", value=lambda reward: reward.value) # Boolean rate metrics (0/1 distributions — ``.mean()`` is the rate) @property diff --git a/tests/unit/orchestrator/test_advantage.py b/tests/unit/orchestrator/test_advantage.py index 153a08bc2f3..247184fee6d 100644 --- a/tests/unit/orchestrator/test_advantage.py +++ b/tests/unit/orchestrator/test_advantage.py @@ -101,7 +101,7 @@ def _take(n: int) -> list[int]: task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt=None)), nodes=nodes, calls=calls, - rewards={"reward": reward}, + rewards={"reward": vf.Reward(score=reward)}, metrics=metrics or {}, ) rollout.env_name = env_name diff --git a/tests/unit/orchestrator/test_algorithms.py b/tests/unit/orchestrator/test_algorithms.py index 7e7a4271126..45e096a859c 100644 --- a/tests/unit/orchestrator/test_algorithms.py +++ b/tests/unit/orchestrator/test_algorithms.py @@ -258,7 +258,7 @@ def _two_turn_rollout(observation_role: str = "tool") -> Rollout: rollout = Rollout( task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt=None)), nodes=nodes, - rewards={"r": 1.0}, + rewards={"r": vf.Reward(score=1.0)}, env_name="test-env", ) rollout.samples = trace_to_samples(rollout, env_name="test-env") @@ -309,7 +309,7 @@ def test_echo_weights_only_content_tokens_when_is_content_present(): rollout = Rollout( task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt=None)), nodes=nodes, - rewards={"r": 1.0}, + rewards={"r": vf.Reward(score=1.0)}, env_name="test-env", ) rollout.samples = trace_to_samples(rollout, env_name="test-env") diff --git a/tests/unit/orchestrator/test_filters.py b/tests/unit/orchestrator/test_filters.py index d2a0489ac31..d858b66153d 100644 --- a/tests/unit/orchestrator/test_filters.py +++ b/tests/unit/orchestrator/test_filters.py @@ -58,7 +58,9 @@ def _make_rollout( else: nodes = [_assistant_node(completion_ids, completion_logprobs)] rollout = Rollout[vf.TaskData]( - task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="")), nodes=nodes, rewards={"reward": reward} + task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="")), + nodes=nodes, + rewards={"reward": vf.Reward(score=reward)}, ) rollout.env_name = "test" rollout.group_id = uuid.uuid4() @@ -140,7 +142,7 @@ def test_gibberish_aligns_logprobs_under_generation_prompt_scaffold(): rollout = Rollout[vf.TaskData]( task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="")), nodes=[_scaffold_assistant_node([50, 80, 120_000], [-1.0, -0.5, gibberish_filter.logprob_threshold - 1.0])], - rewards={"reward": 1.0}, + rewards={"reward": vf.Reward(score=1.0)}, ) result = gibberish_filter.check(rollout) diff --git a/tests/unit/orchestrator/test_metrics.py b/tests/unit/orchestrator/test_metrics.py index 3b417f56919..a3d772046ee 100644 --- a/tests/unit/orchestrator/test_metrics.py +++ b/tests/unit/orchestrator/test_metrics.py @@ -2,6 +2,7 @@ from types import SimpleNamespace import pytest +import verifiers.v1 as vf from prime_rl.orchestrator.metrics import EvalRollouts, Stat, TrainRollouts from prime_rl.orchestrator.utils import compute_pass_metrics @@ -141,8 +142,8 @@ def test_stop_condition_breakdown(): def test_nested_metrics_and_rewards(): rollouts = [ - mk(metrics={"acc": 1.0}, rewards={"correct": 1.0, "format": 0.0}), - mk(metrics={"acc": 3.0, "fmt": 5.0}, rewards={"correct": 0.0, "format": 1.0}), + mk(metrics={"acc": 1.0}, rewards={"correct": vf.Reward(score=1.0), "format": vf.Reward(score=0.0)}), + mk(metrics={"acc": 3.0, "fmt": 5.0}, rewards={"correct": vf.Reward(score=0.0), "format": vf.Reward(score=1.0)}), ] m = TrainRollouts(rollouts).metrics assert m.metrics["acc"].mean() == 2.0 and m.rewards["correct"].mean() == 0.5 # nested group access diff --git a/uv.lock b/uv.lock index 36e4694c649..336cfbd8992 100644 --- a/uv.lock +++ b/uv.lock @@ -67,6 +67,7 @@ members = [ "i3-science-v1", "ifbench-v1", "ifeval-v1", + "kuhn-poker-v1", "lean-v1", "livecodebench-v1", "longbenchpro-v1", @@ -3229,6 +3230,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/2c/5c160dbdef7123f8cc97fd8ece7e0198627a426a2a49614845e9086feb8d/kubernetes-36.0.2-py2.py3-none-any.whl", hash = "sha256:faf9b5241b58de0c4a5069f2a0ffc8ac06fece7215156cd3d3ba081a78a858b6", size = 4617568, upload-time = "2026-06-01T18:20:28.737Z" }, ] +[[package]] +name = "kuhn-poker-v1" +version = "0.1.0" +source = { editable = "deps/verifiers/environments/kuhn_poker_v1" } +dependencies = [ + { name = "verifiers" }, +] + +[package.metadata] +requires-dist = [{ name = "verifiers" }] + [[package]] name = "langdetect" version = "1.0.9" @@ -5325,7 +5337,7 @@ requires-dist = [ [[package]] name = "prime-sandboxes" -version = "0.2.32" +version = "0.2.33" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, @@ -5335,9 +5347,9 @@ dependencies = [ { name = "pydantic" }, { name = "tenacity" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/fd/f57ad87b273ae1696ca620deaac2c56e28a54905d2d2791a5b9a387f443c/prime_sandboxes-0.2.32.tar.gz", hash = "sha256:ea3d6230cdfb5af2e4542814996257bad60fc9913f15f776a65ca1ef05f4f504", size = 77656, upload-time = "2026-07-17T17:41:02.296Z" } +sdist = { url = "https://files.pythonhosted.org/packages/15/a9/880685cefd503aa92d2b49da46b60ba807015f8839c344a83d8c5bc02f30/prime_sandboxes-0.2.33.tar.gz", hash = "sha256:4253a3c345ccf6c07b41a81790f8515763333f094d20bc405a257432461e81d8", size = 81228, upload-time = "2026-07-22T23:23:55.715Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/75/a3/c345509abf8f7707c0c3e9d6badba2e057781d1c05a88a46733ab67521e5/prime_sandboxes-0.2.32-py3-none-any.whl", hash = "sha256:4fe730622e51c489f8f86e7ad959fd8320db6d7a9fe22442ae3c680687da4c22", size = 40897, upload-time = "2026-07-17T17:41:00.935Z" }, + { url = "https://files.pythonhosted.org/packages/f8/1b/10b1044b8aaef561c32a34140f4a13c7310f73f92f9546337e2e8e6a5239/prime_sandboxes-0.2.33-py3-none-any.whl", hash = "sha256:92c9647557e76191ba926ac8ed01708de9ec4450142131b673cd5cf8d9d7ea56", size = 42773, upload-time = "2026-07-22T23:23:54.381Z" }, ] [[package]] @@ -7859,7 +7871,7 @@ requires-dist = [ { name = "openai-agents", specifier = ">=0.8.2" }, { name = "openenv", marker = "extra == 'openenv'", specifier = ">=0.4.1" }, { name = "prime-pydantic-config", extras = ["toml"], specifier = ">=0.4.2" }, - { name = "prime-sandboxes", specifier = ">=0.2.31" }, + { name = "prime-sandboxes", specifier = ">=0.2.33" }, { name = "prime-tunnel", specifier = ">=0.1.8" }, { name = "pydantic", specifier = ">=2.12.3" }, { name = "python-dotenv", marker = "extra == 'browser'", specifier = ">=1.0.0" }, @@ -7902,6 +7914,7 @@ examples = [ { name = "deepwiki-v1", editable = "deps/verifiers/environments/deepwiki_v1" }, { name = "glossary-v1", editable = "deps/verifiers/environments/glossary_v1" }, { name = "gsm8k-v1", editable = "deps/verifiers/environments/gsm8k_v1" }, + { name = "kuhn-poker-v1", editable = "deps/verifiers/environments/kuhn_poker_v1" }, { name = "openenv-wordle-v1", editable = "deps/verifiers/environments/openenv_wordle_v1" }, { name = "proposer-solver-v1", editable = "deps/verifiers/environments/proposer_solver_v1" }, { name = "reverse-text-v1", editable = "deps/verifiers/environments/reverse_text_v1" }, From ca0845cf3f53f2a9742ac7be62af210b248e8e71 Mon Sep 17 00:00:00 2001 From: hallerite Date: Mon, 27 Jul 2026 14:21:49 +0000 Subject: [PATCH 02/10] fix(tests): adopt Reward records in prime-monitor rollout builder Co-Authored-By: Claude Fable 5 --- tests/unit/utils/test_prime_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/utils/test_prime_monitor.py b/tests/unit/utils/test_prime_monitor.py index dc5f0615b76..f6d71cc3882 100644 --- a/tests/unit/utils/test_prime_monitor.py +++ b/tests/unit/utils/test_prime_monitor.py @@ -38,7 +38,7 @@ def _build_rollout(*, example_id: int, reward: float, task: str) -> Rollout: rollout = Rollout[vf.TaskData]( task=vf.TraceTask(type="Task", data=vf.TaskData(idx=example_id, prompt=f"prompt-{example_id}")), nodes=nodes, - rewards={"reward": reward}, + rewards={"reward": vf.Reward(score=reward)}, ) rollout.env_name = task # Per-token advantage stream (full-length-N): 0.0 on the 3 prompt tokens, From 3771bc2263509a7c4420e65dc40b51b04b59be47 Mon Sep 17 00:00:00 2001 From: hallerite Date: Fri, 24 Jul 2026 14:17:02 +0000 Subject: [PATCH 03/10] feat(orchestrator): hierarchical GRPO for proposer-solver envs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the `hierarchical_grpo` algorithm: GRPO baselines at two levels of a task-generating env's episode tree. A group is group_size episodes of the same source task (the proposer asked group_size times); each episode holds the proposer's trace plus the n solver traces its minted task fanned out to. Agents in `episode_agents` (the solvers) mean-center against their own episode's same-agent traces — sibling attempts at the same minted task — and every other agent (the proposer) against its same-agent traces across the group. Rewards never mix across agents or minted tasks; singleton peer sets center to zero. configs/debug/algo/hierarchical_grpo.toml runs proposer-solver-v1 with Qwen3-4B (4 proposals per topic x 4 solvers per proposal, learnability reward on the proposer), both sides trainable. Co-Authored-By: Claude Fable 5 --- configs/debug/algo/hierarchical_grpo.toml | 57 +++++++++++++++ docs/algorithms.md | 33 ++++++++- docs/training.md | 3 +- .../src/prime_rl/configs/algorithm.py | 32 ++++++++- skills/configs/SKILL.md | 2 +- src/prime_rl/orchestrator/algo/__init__.py | 8 ++- .../orchestrator/algo/hierarchical_grpo.py | 49 +++++++++++++ tests/unit/orchestrator/test_advantage.py | 70 +++++++++++++++++++ 8 files changed, 248 insertions(+), 6 deletions(-) create mode 100644 configs/debug/algo/hierarchical_grpo.toml create mode 100644 src/prime_rl/orchestrator/algo/hierarchical_grpo.py diff --git a/configs/debug/algo/hierarchical_grpo.toml b/configs/debug/algo/hierarchical_grpo.toml new file mode 100644 index 00000000000..0518432c892 --- /dev/null +++ b/configs/debug/algo/hierarchical_grpo.toml @@ -0,0 +1,57 @@ +max_steps = 20 +seq_len = 8192 + +[model] +name = "Qwen/Qwen3-4B" + +[wandb] +project = "algorithms-debug" +name = "debug-hierarchical-grpo" + +[orchestrator] +batch_size = 32 +group_size = 4 + +[orchestrator.algo] +type = "hierarchical_grpo" +episode_agents = ["solver"] + +[orchestrator.renderer] +name = "prime-qwen3" + +[orchestrator.train.sampling] +max_completion_tokens = 4096 + +[[orchestrator.train.env]] +name = "proposer-solver" +env.taskset = { id = "proposer-solver-v1" } +env.n = 4 +env.proposer.harness = { id = "null" } +env.proposer.runtime = { type = "subprocess" } +env.solver.harness = { id = "null" } +env.solver.runtime = { type = "subprocess" } + +[orchestrator.eval] +interval = 1 +num_examples = 6 + +[orchestrator.eval.sampling] +temperature = 1.0 +max_completion_tokens = 4096 + +[[orchestrator.eval.env]] +name = "proposer-solver" +env.taskset = { id = "proposer-solver-v1" } +env.n = 4 +env.proposer.harness = { id = "null" } +env.proposer.runtime = { type = "subprocess" } +env.solver.harness = { id = "null" } +env.solver.runtime = { type = "subprocess" } + +[trainer.optim] +lr = 3e-6 + +[ckpt] + +[inference] +gpu_memory_utilization = 0.5 diff --git a/docs/algorithms.md b/docs/algorithms.md index 8088d28516f..e43dc56a566 100644 --- a/docs/algorithms.md +++ b/docs/algorithms.md @@ -17,6 +17,7 @@ This page covers the math and the configurable algorithmic components: the algor - [Custom Loss](#custom-loss) - [Advantage](#advantage) - [Default Advantage](#default-advantage) + - [Hierarchical Advantage (Proposer–Solver)](#hierarchical-advantage-proposersolver) - [Authoring an Algorithm](#authoring-an-algorithm) - [Reference Scoring](#reference-scoring) - [Filters](#filters) @@ -67,6 +68,7 @@ type = "grpo" # the default |---|---|---|---| | `grpo` | policy | `rl` on actions | Standard group-relative RL. | | `max_rl` | policy | `rl` on actions | MaxRL ([arXiv:2602.02710](https://arxiv.org/abs/2602.02710)): GRPO's centered reward normalized by the group **mean** instead of the standard deviation — the gradient is unbiased for the order-`group_size` truncation of the maximum-likelihood objective, upweighting hard examples like `1/p`. | +| `hierarchical_grpo` | policy | `rl` on actions | Two-level GRPO for task-generating envs (proposer-solver): agents in `episode_agents` baseline within their own episode (sibling attempts at the same minted task), every other agent across the group (parallel attempts at the same source task). Needs `episode_agents`. See [Hierarchical Advantage](#hierarchical-advantage-proposersolver). | | `opd` | policy | `ref_kl` on actions | On-policy distillation ([Thinking Machines](https://thinkingmachines.ai/blog/on-policy-distillation/)): the policy samples, per-token reverse KL against a reference model as the gradient signal. Needs a `teacher`. | | `sft` | *(the teacher)* | `ce` on actions | Hard distillation: a frozen model generates rollouts, the policy trains with CE on its tokens. Needs a frozen `sampling.source` (the teacher it samples from). | | `opsd` | policy | `ref_kl` on actions | SDFT ([arXiv:2601.19897](https://arxiv.org/abs/2601.19897)): the model is its own reference, conditioned on an expert demonstration. The teacher *is* the live policy (the paper's setting, no extra deployment) — no model to configure. | @@ -140,6 +142,7 @@ At runtime, each env's resolved config builds two objects: a `Sampler` (`prime_r | `grpo` | `GRPOAlgorithm` | `score_group`: group-norm credit (optional length penalty) | | `echo` | `EchoAlgorithm` | `score_rollout`: weighted ce on observation tokens; `score_group`: group-norm credit (inherited) | | `max_rl` | `MaxRLAlgorithm` | `score_group`: mean-normalized group credit | +| `hierarchical_grpo` | `HierarchicalGRPOAlgorithm` | `score_group`: two-level peer-set credit (per-episode / per-group) | | `opd` | `OPDAlgorithm` | `score_rollout`: own-context prefill under the teacher | | `opsd` | `OPSDAlgorithm` | `score_rollout`: demo-conditioned prefill under the live policy | | `sft` | `SFTDistillAlgorithm` | `score_group`: group-norm credit (feeds filters) | @@ -176,7 +179,7 @@ $$ \mathcal{L} = \frac{\sum \mathcal{L}_{rl}}{N_{rl}} + \frac{\sum \mathcal{L}_{ce}}{N_{ce}} + \frac{\sum \mathcal{L}_{ref\_kl}}{N_{ref\_kl}} $$ -- `rl` — the configured RL loss (`[trainer.loss]`): DPPO + KL by default, or a [custom loss](#custom-loss). Fed by the group-relative algorithms (`grpo`, `max_rl`, and `echo`'s action tokens). +- `rl` — the configured RL loss (`[trainer.loss]`): DPPO + KL by default, or a [custom loss](#custom-loss). Fed by the advantage-assigning algorithms (`grpo`, `max_rl`, `hierarchical_grpo`, and `echo`'s action tokens). - `ce` — masked NLL. Used for frozen-model tokens (`sft`) and env-observation tokens (`echo`). - `ref_kl` — the per-token reverse KL to a reference model ($\log \pi_{\text{ref}} - \log \pi$) as the policy-gradient signal, importance-ratio corrected with a one-sided trust region (`opd`, `opsd`). Requires `ref_logprobs` from a [reference scoring](#reference-scoring); the scoring model must be a vLLM server (it's the only one that exposes `prompt_logprobs`). @@ -278,6 +281,7 @@ The per-token training signal is set by `algo.type` and the [algorithm](#the-alg |---|---|---| | `grpo` | `rl` | Group-norm: reward minus per-group baseline, optional length penalty. | | `max_rl` | `rl` | Mean-normalized group credit (maximum-likelihood RL). | +| `hierarchical_grpo` | `rl` | Two-level peer-set credit for task-generating envs: episode-scoped agents center within their episode, the rest across the group. | | `echo` | `rl` + `ce` | Group-norm on action tokens, plus weighted CE on env-provided tokens selected by message role (each role's `alpha` is its ECHO λ), optionally narrowed by a user filter. | | `opd` | `ref_kl` | On-policy distillation: per-token reverse KL to a reference model (`teacher`, an inline frozen hosted model), evaluated in the trainer from shipped reference logprobs. No credit — rollouts keep `advantages = None` (advantage-based filters never fire) and ship no advantage stream; `group_size` only fans out sampling. | | `opsd` | `ref_kl` | SDFT: per-token reverse KL to a demo-conditioned reference. No credit — rollouts keep `advantages = None` (advantage-based filters never fire) and ship no advantage stream. | @@ -299,6 +303,33 @@ type = "grpo" type = "linear" ``` +### Hierarchical Advantage (Proposer–Solver) + +Task-generating envs break the flat-group assumption twice over. In `proposer-solver-v1`, one episode is a proposer trace plus the n solver traces its minted problem fanned out to, and a group is `group_size` episodes of the same source task — the proposer asked `group_size` times to invent a problem. Pooling solver rewards across episodes baselines them against attempts at *different* problems; pooling proposer and solver traces mixes reward scales across agents (the proposer earns learnability, the solvers correctness). + +`hierarchical_grpo` computes GRPO baselines at two levels of that episode tree. Every trainable trace is mean-centered against its *peer set*: + +- agents listed in `episode_agents` (the solvers) against the same-agent traces of **their own episode** — sibling attempts at the same minted task; +- every other agent (the proposer) against its same-agent traces **across the group** — parallel attempts at the same source task, each rewarded by what its minted task did to the solvers (`proposer-solver-v1`'s learnability, `4p(1−p)`, peaks when half the solvers crack the problem). + +Rewards never mix across agents or across minted tasks. A peer set of one centers to zero advantage (GRPO's singleton convention; the zero-advantage filter drops it). `episode_agents` is required — which agents are episode-scoped is env truth the algorithm must not guess: + +```toml +[orchestrator.algo] +type = "hierarchical_grpo" +episode_agents = ["solver"] + +[[orchestrator.train.env]] +name = "proposer-solver" +env.taskset = { id = "proposer-solver-v1" } +env.n = 4 # solvers per proposed problem +env.proposer.harness = { id = "null" } +env.proposer.runtime = { type = "subprocess" } +env.solver.harness = { id = "null" } +env.solver.runtime = { type = "subprocess" } +``` + +Here `group_size` is the number of proposals per source task and `env.n` the solvers per proposal, so one group carries `group_size × (1 + n)` traces. To train only one side, flip the env's own switches (`env.train_proposer` / `env.train_solver`) — the untrainable side's traces never reach the advantage computation. ### Authoring an Algorithm diff --git a/docs/training.md b/docs/training.md index ce2bacae6db..3698b3a76af 100644 --- a/docs/training.md +++ b/docs/training.md @@ -59,7 +59,7 @@ A condensed view of the knobs you'll most often tune. For trainer-side paralleli | `orchestrator.batch_size` | Tasks per trainer step. | | `orchestrator.group_size` | Rollouts generated per task. | | `orchestrator.max_off_policy_steps` | How many distinct policies may have contributed to one rollout before it's discarded (default 8). The main off-policy dial on long agentic rollouts — bump for throughput, lower for tighter on-policyness. Watch `errored_rollouts` and `mismatch_kl/all/mean` when tuning. | -| `[orchestrator.algo]` | Training algorithm — its `type` names it (`grpo` default, `max_rl`, `opd`, `opsd`, `sft`, `echo`). See [Algorithms](#algorithms). | +| `[orchestrator.algo]` | Training algorithm — its `type` names it (`grpo` default, `max_rl`, `hierarchical_grpo`, `opd`, `opsd`, `sft`, `echo`). See [Algorithms](#algorithms). | | `[[orchestrator.train.env]]` | Training environments. List multiple tables for multi-env training; weight them via `ratio`. See [Configuration § Environments](configuration.md#environments-orchestratortrainenv). | | `[[orchestrator.eval.env]]` + `orchestrator.eval.interval` | Eval environments and cadence (default every 100 steps). | @@ -89,6 +89,7 @@ The RL entrypoint supports several training algorithms, switched via `[orchestra |---|---|---| | `grpo` (default) | None | Standard group-relative RL | | `max_rl` | None | [MaxRL](https://arxiv.org/abs/2602.02710): GRPO with mean-normalized advantages (maximum-likelihood RL) | +| `hierarchical_grpo` | None | Two-level GRPO for task-generating envs (e.g. `proposer-solver-v1`): episode-scoped agents baseline within their episode, the rest across the group | | `opd` | Required, must be vLLM (needs `prompt_logprobs`) | [On-policy distillation](https://thinkingmachines.ai/blog/on-policy-distillation/): the policy generates rollouts, the trainer minimizes per-token reverse KL to a reference model | | `sft` | Required, any OpenAI-compatible endpoint | Hard-distill: a frozen model generates rollouts, the policy trains on its tokens | | `opsd` | None — the live policy is its own reference (no deployment) | [SDFT](https://arxiv.org/abs/2601.19897): the model is its own reference conditioned on expert demonstrations | diff --git a/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py b/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py index b3282475afd..41e7f8cb7c1 100644 --- a/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py +++ b/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py @@ -232,6 +232,29 @@ class MaxRLAlgoConfig(BaseAlgoConfig): action_loss_type: ClassVar[ActionLossType] = "rl" +class HierarchicalGRPOAlgoConfig(BaseAlgoConfig): + type: Literal["hierarchical_grpo"] = "hierarchical_grpo" + """Hierarchical GRPO for task-generating envs (``proposer-solver-v1`` and + friends): GRPO baselines computed at two levels of the episode tree. A + group is ``group_size`` episodes of the same source task, and one episode + holds a proposer trace plus the n solver traces its minted task fanned out + to. Every trainable trace is mean-centered against its *peer set*: agents + listed in ``episode_agents`` (the solvers) against the same-agent traces of + their own episode — sibling attempts at the same minted task — and every + other agent (the proposer) against its same-agent traces across the group — + parallel attempts at the same source task. Rewards never mix across agents + or across minted tasks.""" + + action_loss_type: ClassVar[ActionLossType] = "rl" + + episode_agents: list[str] = Field(min_length=1) + """Agents whose peer set is their own episode (``["solver"]`` for + ``proposer-solver-v1``): their reward compares only against sibling traces + of the same agent in the same episode, because each episode mints its own + task and rewards are not exchangeable across episodes. Required — which + agents are episode-scoped is env truth the algorithm must not guess.""" + + class OPDAlgoConfig(BaseAlgoConfig): type: Literal["opd"] = "opd" """On-policy distillation: the per-token signal is the reverse KL to @@ -305,7 +328,13 @@ def require_frozen_source(self): AlgoConfig: TypeAlias = Annotated[ - GRPOAlgoConfig | EchoAlgoConfig | MaxRLAlgoConfig | OPDAlgoConfig | OPSDAlgoConfig | SFTAlgoConfig, + GRPOAlgoConfig + | EchoAlgoConfig + | MaxRLAlgoConfig + | HierarchicalGRPOAlgoConfig + | OPDAlgoConfig + | OPSDAlgoConfig + | SFTAlgoConfig, Field(discriminator="type"), ] """The training algorithm: sampling plus the per-token training signal (credit @@ -314,6 +343,7 @@ def require_frozen_source(self): - ``grpo`` — policy group sampling, group-relative advantage, RL loss (the default). - ``max_rl`` — GRPO with mean-normalized advantages (maximum-likelihood RL). +- ``hierarchical_grpo`` — two-level GRPO for task-generating envs: episode-scoped agents baseline within their episode, the rest across the group. Needs ``episode_agents``. - ``opd`` — on-policy distillation: policy samples, per-token reverse KL against a reference model. Needs ``teacher``. - ``opsd`` — SDFT: policy samples, demo-conditioned reverse KL against the live policy (the teacher is the policy itself). - ``sft`` — a frozen model samples, the policy trains with CE on its tokens. Needs a frozen ``sampling.source``. diff --git a/skills/configs/SKILL.md b/skills/configs/SKILL.md index 81535642770..55d7689162f 100644 --- a/skills/configs/SKILL.md +++ b/skills/configs/SKILL.md @@ -54,7 +54,7 @@ CLI: `--orchestrator.train.env.0.env.taskset.id reverse-text-v1`. **Discriminated unions** — set the `type` field to pick the variant (`[orchestrator.algo] type = "max_rl"`). Omit `type` to keep the default variant. -**Algorithms** — `[orchestrator.algo] type = "grpo" | "max_rl" | "opd" | "opsd" | "sft" | "echo"` — the type names the algorithm (credit assignment + loss routing, fused), and each type's class defaults are its vetted setting; any other key you set is your own assembly (e.g. `[orchestrator.algo.roles.user] alpha = 0.1` for echo — setting any echo role replaces the whole role table). There is no preset layer, and no config hook that points at user code — a new algorithm is a named class in the repo (subclass `Algorithm`, register it). Per-env override: `[orchestrator.train.env.algo] type = "opd"` (the env assembles its own algorithm). prime-rl only hosts the trainable policy; frozen models are inline external endpoints on the algorithm, named where the model is used — `[orchestrator.algo.teacher]` for opd (the frozen model scored against), `[orchestrator.algo.sampling.source]` for sft (the model it samples from), each with `name` + `base_url`. There is no shared `teacher` slot. opsd declares no model — it self-distills against the live policy. See `docs/algorithms.md`. +**Algorithms** — `[orchestrator.algo] type = "grpo" | "max_rl" | "hierarchical_grpo" | "opd" | "opsd" | "sft" | "echo"` — the type names the algorithm (credit assignment + loss routing, fused), and each type's class defaults are its vetted setting; any other key you set is your own assembly (e.g. `[orchestrator.algo.roles.user] alpha = 0.1` for echo — setting any echo role replaces the whole role table). There is no preset layer, and no config hook that points at user code — a new algorithm is a named class in the repo (subclass `Algorithm`, register it). Per-env override: `[orchestrator.train.env.algo] type = "opd"` (the env assembles its own algorithm). prime-rl only hosts the trainable policy; frozen models are inline external endpoints on the algorithm, named where the model is used — `[orchestrator.algo.teacher]` for opd (the frozen model scored against), `[orchestrator.algo.sampling.source]` for sft (the model it samples from), each with `name` + `base_url`. There is no shared `teacher` slot. opsd declares no model — it self-distills against the live policy. See `docs/algorithms.md`. **`BaseModel | None` fields** — bare flag enables defaults; nested override enables and sets: diff --git a/src/prime_rl/orchestrator/algo/__init__.py b/src/prime_rl/orchestrator/algo/__init__.py index 8d1baa60a33..01947ca9a30 100644 --- a/src/prime_rl/orchestrator/algo/__init__.py +++ b/src/prime_rl/orchestrator/algo/__init__.py @@ -5,8 +5,9 @@ turns the signal half into runtime objects (the sampling half is the env's :class:`~prime_rl.orchestrator.sampler.Sampler`): -- one module per algorithm (``grpo``, ``echo``, ``max_rl``, ``opd``, - ``opsd``, ``sft``) — each named class owns its scoring hooks +- one module per algorithm (``grpo``, ``echo``, ``max_rl``, + ``hierarchical_grpo``, ``opd``, ``opsd``, ``sft``) — each named class owns + its scoring hooks (``score_rollout`` / ``score_group``) and declares what it needs (loss component, a "teacher", ...). One instance per env, built by :func:`build_algorithm`. A new credit-assignment scheme is a new named class: @@ -31,6 +32,7 @@ from prime_rl.orchestrator.algo.base import Algorithm, connect_frozen_pool from prime_rl.orchestrator.algo.echo import EchoAlgorithm from prime_rl.orchestrator.algo.grpo import GRPOAlgorithm +from prime_rl.orchestrator.algo.hierarchical_grpo import HierarchicalGRPOAlgorithm from prime_rl.orchestrator.algo.max_rl import MaxRLAlgorithm from prime_rl.orchestrator.algo.opd import OPDAlgorithm from prime_rl.orchestrator.algo.opsd import OPSDAlgorithm @@ -48,6 +50,7 @@ "grpo": GRPOAlgorithm, "echo": EchoAlgorithm, "max_rl": MaxRLAlgorithm, + "hierarchical_grpo": HierarchicalGRPOAlgorithm, "opd": OPDAlgorithm, "opsd": OPSDAlgorithm, "sft": SFTDistillAlgorithm, @@ -69,6 +72,7 @@ def build_algorithm(config: AlgoConfig, policy_pool: InferencePool) -> Algorithm "Algorithm", "EchoAlgorithm", "GRPOAlgorithm", + "HierarchicalGRPOAlgorithm", "MaxRLAlgorithm", "OPDAlgorithm", "OPSDAlgorithm", diff --git a/src/prime_rl/orchestrator/algo/hierarchical_grpo.py b/src/prime_rl/orchestrator/algo/hierarchical_grpo.py new file mode 100644 index 00000000000..0debde3126b --- /dev/null +++ b/src/prime_rl/orchestrator/algo/hierarchical_grpo.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from collections import defaultdict +from typing import TYPE_CHECKING + +from prime_rl.configs.algorithm import HierarchicalGRPOAlgoConfig +from prime_rl.orchestrator.algo.base import Algorithm + +if TYPE_CHECKING: + from prime_rl.orchestrator.types import Rollout + from prime_rl.utils.client import InferencePool + + +class HierarchicalGRPOAlgorithm(Algorithm): + """Hierarchical GRPO for task-generating envs (proposer-solver-v1 and + friends): GRPO baselines computed at two levels of the episode tree. + + A group is ``group_size`` episodes of the same source task — the proposer + asked ``group_size`` times to invent a problem — and each episode holds the + proposer's trace plus the n solver traces its minted task fanned out to. + Neither level is safe under plain GRPO: pooling solver rewards across + episodes baselines them against attempts at *different* problems, and + pooling proposer and solver traces mixes reward scales across agents. + + Every trainable trace is instead mean-centered against its peer set: + + - agents in ``episode_agents`` (the solvers) against the same-agent traces + of their own episode — sibling attempts at the same minted task; + - every other agent (the proposer) against its same-agent traces across + the group — parallel attempts at the same source task, rewarded by what + their minted tasks did to the solvers (e.g. learnability). + + A peer set of one centers to zero advantage (GRPO's singleton convention; + the zero-advantage filter drops it).""" + + def __init__(self, config: HierarchicalGRPOAlgoConfig, policy_pool: InferencePool): + super().__init__(config, policy_pool) + self.episode_agents = set(config.episode_agents) + + async def score_group(self, group: list[Rollout]) -> None: + peers: dict[tuple[str | None, str | None], list[Rollout]] = defaultdict(list) + for rollout in group: + episode_scoped = rollout.agent_name in self.episode_agents + key = (rollout.agent_name, rollout.episode_id if episode_scoped else None) + peers[key].append(rollout) + for members in peers.values(): + baseline = sum(rollout.reward for rollout in members) / len(members) + for rollout in members: + rollout.assign_advantages(rollout.reward - baseline) diff --git a/tests/unit/orchestrator/test_advantage.py b/tests/unit/orchestrator/test_advantage.py index 247184fee6d..1092234d533 100644 --- a/tests/unit/orchestrator/test_advantage.py +++ b/tests/unit/orchestrator/test_advantage.py @@ -5,10 +5,12 @@ from prime_rl.configs.algorithm import ( GRPOAlgoConfig, + HierarchicalGRPOAlgoConfig, LinearLengthPenaltyConfig, MaxRLAlgoConfig, ) from prime_rl.orchestrator.algo.grpo import GRPOAlgorithm +from prime_rl.orchestrator.algo.hierarchical_grpo import HierarchicalGRPOAlgorithm from prime_rl.orchestrator.algo.max_rl import MaxRLAlgorithm from prime_rl.orchestrator.trajectories import trace_to_samples from prime_rl.orchestrator.types import Rollout @@ -183,6 +185,74 @@ def test_max_rl_mean_normalized(): assert _max_rl(_make_group(rewards=[1.0, 1.0])) == pytest.approx([0.0, 0.0]) +# -------------------------------------------------------------------------- +# Hierarchical GRPO: two-level baselines for task-generating envs. +# -------------------------------------------------------------------------- + + +def _episode_rollout(reward: float, agent_name: str, episode_id: str) -> Rollout: + """A one-turn rollout stamped as one agent's trace of one episode — what a + task-generating env's traces look like by the time ``score_group`` sees them.""" + rollout = _build_rollout(reward, sampled_lengths=[2]) + rollout.agent = vf.AgentInfo(config=vf.AgentConfig(), name=agent_name) + rollout.episode_id = episode_id + return rollout + + +def _hier_grpo(group: list[Rollout], episode_agents: list[str]) -> list[float]: + """Drive ``HierarchicalGRPOAlgorithm.score_group`` and read back the scalars.""" + algo = HierarchicalGRPOAlgorithm(HierarchicalGRPOAlgoConfig(episode_agents=episode_agents), policy_pool=None) + asyncio.run(algo.score_group(group)) + return [_scalar(rollout) for rollout in group] + + +def test_hier_grpo_solvers_center_within_their_episode(): + """Solvers are baselined only against siblings attempting the same minted + task: an episode whose problem got 1-of-2 solved carries signal, while + pooling all four solver rewards (mean 0.25) would mis-credit both episodes.""" + group = [ + _episode_rollout(1.0, "solver", "ep1"), + _episode_rollout(0.0, "solver", "ep1"), + _episode_rollout(0.0, "solver", "ep2"), + _episode_rollout(0.0, "solver", "ep2"), + ] + advs = _hier_grpo(group, episode_agents=["solver"]) + assert advs[:2] == pytest.approx([0.5, -0.5]) + assert advs[2:] == pytest.approx([0.0, 0.0]) + + +def test_hier_grpo_proposers_center_across_the_group(): + """The proposer's peer set is its own traces across the group's episodes — + solver rewards never leak into its baseline, nor its into theirs.""" + group = [ + _episode_rollout(1.0, "proposer", "ep1"), + _episode_rollout(1.0, "solver", "ep1"), + _episode_rollout(0.0, "proposer", "ep2"), + _episode_rollout(1.0, "solver", "ep2"), + ] + advs = _hier_grpo(group, episode_agents=["solver"]) + assert advs[0] == pytest.approx(0.5) + assert advs[2] == pytest.approx(-0.5) + # each episode's lone solver is a singleton peer set: zero advantage + assert advs[1] == pytest.approx(0.0) + assert advs[3] == pytest.approx(0.0) + + +def test_hier_grpo_proposer_solver_shape(): + """The proposer-solver-v1 shape end to end: 2 proposals x 2 solvers, all + peer sets centered independently and each summing to zero.""" + group = [ + _episode_rollout(1.0, "proposer", "ep1"), + _episode_rollout(1.0, "solver", "ep1"), + _episode_rollout(0.0, "solver", "ep1"), + _episode_rollout(0.0, "proposer", "ep2"), + _episode_rollout(1.0, "solver", "ep2"), + _episode_rollout(1.0, "solver", "ep2"), + ] + advs = _hier_grpo(group, episode_agents=["solver"]) + assert advs == pytest.approx([0.5, 0.5, -0.5, -0.5, 0.0, 0.0]) + + # -------------------------------------------------------------------------- # GRPO linear length penalty: pass_rate-scaled penalty before the baseline. # -------------------------------------------------------------------------- From ef1a166afd90fa9097592a0511de2ea1ff61a65b Mon Sep 17 00:00:00 2001 From: hallerite Date: Wed, 29 Jul 2026 13:43:18 +0000 Subject: [PATCH 04/10] gate hierarchical GRPO on proposer-solver envs, drop its unit tests The two-level peer sets are the proposer-solver episode tree; on a flat env they collapse to singletons and every group trains on zero advantages. Nothing in a rollout reveals the tree, so the env's resolved config class is the check: algorithms get a validate_env hook (no-op on the base) that the orchestrator config calls per train env once algorithm inheritance resolves. Co-Authored-By: Claude Opus 5 (1M context) --- docs/algorithms.md | 2 +- .../src/prime_rl/configs/algorithm.py | 33 ++++++++- .../src/prime_rl/configs/orchestrator.py | 11 +++ .../orchestrator/algo/hierarchical_grpo.py | 5 +- tests/unit/orchestrator/test_advantage.py | 70 ------------------- 5 files changed, 48 insertions(+), 73 deletions(-) diff --git a/docs/algorithms.md b/docs/algorithms.md index cfa135c56da..406721ab9c3 100644 --- a/docs/algorithms.md +++ b/docs/algorithms.md @@ -316,7 +316,7 @@ Task-generating envs break the flat-group assumption twice over. In `proposer-so - agents listed in `episode_agents` (the solvers) against the same-agent traces of **their own episode** — sibling attempts at the same minted task; - every other agent (the proposer) against its same-agent traces **across the group** — parallel attempts at the same source task, each rewarded by what its minted task did to the solvers (`proposer-solver-v1`'s learnability, `4p(1−p)`, peaks when half the solvers crack the problem). -Rewards never mix across agents or across minted tasks. A peer set of one centers to zero advantage (GRPO's singleton convention; the zero-advantage filter drops it). `episode_agents` is required — which agents are episode-scoped is env truth the algorithm must not guess: +Rewards never mix across agents or across minted tasks. A peer set of one centers to zero advantage (GRPO's singleton convention; the zero-advantage filter drops it). Config validation rejects any env that isn't a proposer-solver env — the two-level peer sets *are* that episode tree, and on a flat env they collapse to one trace each, so every group would train on nothing but zeros. `episode_agents` is required — which agents are episode-scoped is env truth the algorithm must not guess: ```toml [orchestrator.algo] diff --git a/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py b/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py index 40a59136478..ff3841db9fc 100644 --- a/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py +++ b/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py @@ -35,6 +35,7 @@ from typing import Annotated, Any, ClassVar, Literal, TypeAlias +import verifiers.v1 as vf from pydantic import Field, model_validator from renderers import AutoRendererConfig, RendererConfig @@ -188,6 +189,13 @@ def validate_sampling_source(self): ) return self + def validate_env(self, env_config: vf.EnvConfig) -> None: + """Raise if this algorithm cannot run on the env it is configured for, + given that env's resolved config. Most algorithms read only the rollouts + and their rewards, so they run on any env and the base does nothing; + override where the credit assignment encodes an env's episode structure. + Called once per train env after algorithm inheritance resolves.""" + class GRPOAlgoConfig(BaseAlgoConfig): type: Literal["grpo"] = "grpo" @@ -264,7 +272,8 @@ class HierarchicalGRPOAlgoConfig(BaseAlgoConfig): their own episode — sibling attempts at the same minted task — and every other agent (the proposer) against its same-agent traces across the group — parallel attempts at the same source task. Rewards never mix across agents - or across minted tasks.""" + or across minted tasks. Only a proposer-solver env may train under it — the + peer sets are that env's episode tree (see ``validate_env``).""" action_loss_type: ClassVar[ActionLossType] = "rl" @@ -275,6 +284,28 @@ class HierarchicalGRPOAlgoConfig(BaseAlgoConfig): task and rewards are not exchangeable across episodes. Required — which agents are episode-scoped is env truth the algorithm must not guess.""" + def validate_env(self, env_config: vf.EnvConfig) -> None: + """Only a proposer-solver env may train under hierarchical GRPO. The + two-level peer sets *are* that env's episode tree — one proposer trace + per episode plus the solver traces its minted task fanned out to — so on + a flat env the levels collapse and every trace silently lands in a peer + set of one (zero advantage, whole group filtered). Nothing in a rollout + reveals the tree, so the env's own config class is the check.""" + try: + from proposer_solver_v1 import ProposerSolverEnvConfig + except ImportError as e: + raise ValueError( + "algorithm 'hierarchical_grpo' requires a proposer-solver env, but the " + "proposer-solver-v1 package is not installed (`uv sync --all-packages`)." + ) from e + if not isinstance(env_config, ProposerSolverEnvConfig): + raise ValueError( + f"algorithm 'hierarchical_grpo' needs a proposer-solver env, but this env resolved to " + f"{type(env_config).__name__} — its two-level baselines read an episode tree " + "(a proposer trace plus the solver traces its minted task fanned out to) that a flat " + "env does not have. Use 'grpo' (or 'rae' for multi-agent self-play)." + ) + class OPDAlgoConfig(BaseAlgoConfig): type: Literal["opd"] = "opd" diff --git a/packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py b/packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py index b0b71fd4b61..d501869a06d 100644 --- a/packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py +++ b/packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py @@ -622,6 +622,17 @@ def inherit_env_algorithms(self): env_cfg.algo = self.algo.model_copy(deep=True) return self + @model_validator(mode="after") + def validate_env_algorithms(self): + """Let each env's resolved algorithm reject the env it was pointed at — + the algorithms whose credit assignment encodes an env's episode structure + (hierarchical_grpo) only make sense on that env, and a flat env would + silently train on collapsed peer sets.""" + for env_cfg in self.train.env: + assert env_cfg.algo is not None # resolved by inherit_env_algorithms + env_cfg.algo.validate_env(env_cfg.env) + return self + @property def any_policy_sourced(self) -> bool: """True when at least one train env samples rollouts from the live policy.""" diff --git a/src/prime_rl/orchestrator/algo/hierarchical_grpo.py b/src/prime_rl/orchestrator/algo/hierarchical_grpo.py index 0debde3126b..e6ad887515e 100644 --- a/src/prime_rl/orchestrator/algo/hierarchical_grpo.py +++ b/src/prime_rl/orchestrator/algo/hierarchical_grpo.py @@ -31,7 +31,10 @@ class HierarchicalGRPOAlgorithm(Algorithm): their minted tasks did to the solvers (e.g. learnability). A peer set of one centers to zero advantage (GRPO's singleton convention; - the zero-advantage filter drops it).""" + the zero-advantage filter drops it). Nothing in a rollout reveals the tree, + so the env itself is gated at config validation + (:meth:`HierarchicalGRPOAlgoConfig.validate_env`) rather than here: on a flat + env every peer set would be a singleton and every group train on zeros.""" def __init__(self, config: HierarchicalGRPOAlgoConfig, policy_pool: InferencePool): super().__init__(config, policy_pool) diff --git a/tests/unit/orchestrator/test_advantage.py b/tests/unit/orchestrator/test_advantage.py index 1092234d533..247184fee6d 100644 --- a/tests/unit/orchestrator/test_advantage.py +++ b/tests/unit/orchestrator/test_advantage.py @@ -5,12 +5,10 @@ from prime_rl.configs.algorithm import ( GRPOAlgoConfig, - HierarchicalGRPOAlgoConfig, LinearLengthPenaltyConfig, MaxRLAlgoConfig, ) from prime_rl.orchestrator.algo.grpo import GRPOAlgorithm -from prime_rl.orchestrator.algo.hierarchical_grpo import HierarchicalGRPOAlgorithm from prime_rl.orchestrator.algo.max_rl import MaxRLAlgorithm from prime_rl.orchestrator.trajectories import trace_to_samples from prime_rl.orchestrator.types import Rollout @@ -185,74 +183,6 @@ def test_max_rl_mean_normalized(): assert _max_rl(_make_group(rewards=[1.0, 1.0])) == pytest.approx([0.0, 0.0]) -# -------------------------------------------------------------------------- -# Hierarchical GRPO: two-level baselines for task-generating envs. -# -------------------------------------------------------------------------- - - -def _episode_rollout(reward: float, agent_name: str, episode_id: str) -> Rollout: - """A one-turn rollout stamped as one agent's trace of one episode — what a - task-generating env's traces look like by the time ``score_group`` sees them.""" - rollout = _build_rollout(reward, sampled_lengths=[2]) - rollout.agent = vf.AgentInfo(config=vf.AgentConfig(), name=agent_name) - rollout.episode_id = episode_id - return rollout - - -def _hier_grpo(group: list[Rollout], episode_agents: list[str]) -> list[float]: - """Drive ``HierarchicalGRPOAlgorithm.score_group`` and read back the scalars.""" - algo = HierarchicalGRPOAlgorithm(HierarchicalGRPOAlgoConfig(episode_agents=episode_agents), policy_pool=None) - asyncio.run(algo.score_group(group)) - return [_scalar(rollout) for rollout in group] - - -def test_hier_grpo_solvers_center_within_their_episode(): - """Solvers are baselined only against siblings attempting the same minted - task: an episode whose problem got 1-of-2 solved carries signal, while - pooling all four solver rewards (mean 0.25) would mis-credit both episodes.""" - group = [ - _episode_rollout(1.0, "solver", "ep1"), - _episode_rollout(0.0, "solver", "ep1"), - _episode_rollout(0.0, "solver", "ep2"), - _episode_rollout(0.0, "solver", "ep2"), - ] - advs = _hier_grpo(group, episode_agents=["solver"]) - assert advs[:2] == pytest.approx([0.5, -0.5]) - assert advs[2:] == pytest.approx([0.0, 0.0]) - - -def test_hier_grpo_proposers_center_across_the_group(): - """The proposer's peer set is its own traces across the group's episodes — - solver rewards never leak into its baseline, nor its into theirs.""" - group = [ - _episode_rollout(1.0, "proposer", "ep1"), - _episode_rollout(1.0, "solver", "ep1"), - _episode_rollout(0.0, "proposer", "ep2"), - _episode_rollout(1.0, "solver", "ep2"), - ] - advs = _hier_grpo(group, episode_agents=["solver"]) - assert advs[0] == pytest.approx(0.5) - assert advs[2] == pytest.approx(-0.5) - # each episode's lone solver is a singleton peer set: zero advantage - assert advs[1] == pytest.approx(0.0) - assert advs[3] == pytest.approx(0.0) - - -def test_hier_grpo_proposer_solver_shape(): - """The proposer-solver-v1 shape end to end: 2 proposals x 2 solvers, all - peer sets centered independently and each summing to zero.""" - group = [ - _episode_rollout(1.0, "proposer", "ep1"), - _episode_rollout(1.0, "solver", "ep1"), - _episode_rollout(0.0, "solver", "ep1"), - _episode_rollout(0.0, "proposer", "ep2"), - _episode_rollout(1.0, "solver", "ep2"), - _episode_rollout(1.0, "solver", "ep2"), - ] - advs = _hier_grpo(group, episode_agents=["solver"]) - assert advs == pytest.approx([0.5, 0.5, -0.5, -0.5, 0.0, 0.0]) - - # -------------------------------------------------------------------------- # GRPO linear length penalty: pass_rate-scaled penalty before the baseline. # -------------------------------------------------------------------------- From f6ff3a1514616e738334ef029bd2a1ef309844ad Mon Sep 17 00:00:00 2001 From: hallerite Date: Wed, 29 Jul 2026 13:44:36 +0000 Subject: [PATCH 05/10] note the hierarchical GRPO env gate in the configs skill Co-Authored-By: Claude Opus 5 (1M context) --- skills/configs/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/configs/SKILL.md b/skills/configs/SKILL.md index 0350c231137..9a1c24cd482 100644 --- a/skills/configs/SKILL.md +++ b/skills/configs/SKILL.md @@ -54,7 +54,7 @@ CLI: `--orchestrator.train.env.0.env.taskset.id reverse-text-v1`. **Discriminated unions** — set the `type` field to pick the variant (`[orchestrator.algo] type = "max_rl"`). Omit `type` to keep the default variant. -**Algorithms** — `[orchestrator.algo] type = "grpo" | "max_rl" | "rae" | "hierarchical_grpo" | "opd" | "opsd" | "sft" | "echo"` — the type names the algorithm (credit assignment + loss routing, fused), and each type's class defaults are its vetted setting; any other key you set is your own assembly (e.g. `[orchestrator.algo.roles.user] alpha = 0.1` for echo — setting any echo role replaces the whole role table). There is no preset layer, and no config hook that points at user code — a new algorithm is a named class in the repo (subclass `Algorithm`, register it). Per-env override: `[orchestrator.train.env.algo] type = "opd"` (the env assembles its own algorithm). prime-rl only hosts the trainable policy; frozen models are inline external endpoints on the algorithm, named where the model is used — `[orchestrator.algo.teacher]` for opd (the frozen model scored against), `[orchestrator.algo.sampling.source]` for sft (the model it samples from), each with `name` + `base_url`. There is no shared `teacher` slot. opsd declares no model — it self-distills against the live policy. See `docs/algorithms.md`. +**Algorithms** — `[orchestrator.algo] type = "grpo" | "max_rl" | "rae" | "hierarchical_grpo" | "opd" | "opsd" | "sft" | "echo"` — the type names the algorithm (credit assignment + loss routing, fused), and each type's class defaults are its vetted setting; any other key you set is your own assembly (e.g. `[orchestrator.algo.roles.user] alpha = 0.1` for echo — setting any echo role replaces the whole role table). An algorithm may constrain its env: `hierarchical_grpo` is rejected at config validation on anything but a proposer-solver env, since its two-level baselines read that episode tree. There is no preset layer, and no config hook that points at user code — a new algorithm is a named class in the repo (subclass `Algorithm`, register it). Per-env override: `[orchestrator.train.env.algo] type = "opd"` (the env assembles its own algorithm). prime-rl only hosts the trainable policy; frozen models are inline external endpoints on the algorithm, named where the model is used — `[orchestrator.algo.teacher]` for opd (the frozen model scored against), `[orchestrator.algo.sampling.source]` for sft (the model it samples from), each with `name` + `base_url`. There is no shared `teacher` slot. opsd declares no model — it self-distills against the live policy. See `docs/algorithms.md`. **`BaseModel | None` fields** — bare flag enables defaults; nested override enables and sets: From fdb2f12000701c2ff108c880562caa184d01ff53 Mon Sep 17 00:00:00 2001 From: hallerite Date: Wed, 29 Jul 2026 13:48:09 +0000 Subject: [PATCH 06/10] reword the hierarchical GRPO summaries to name the two baselines "episode-scoped agents center within their episode, the rest across the group" defined the term with itself and left the other half unnamed. The neighbouring rows all state their math; now that the algorithm is gated to proposer-solver envs these can name the roles outright. Co-Authored-By: Claude Opus 5 (1M context) --- docs/algorithms.md | 4 ++-- docs/training.md | 2 +- packages/prime-rl-configs/src/prime_rl/configs/algorithm.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/algorithms.md b/docs/algorithms.md index 406721ab9c3..630dfc06ff2 100644 --- a/docs/algorithms.md +++ b/docs/algorithms.md @@ -145,7 +145,7 @@ At runtime, each env's resolved config builds two objects: a `Sampler` (`prime_r | `echo` | `EchoAlgorithm` | `score_rollout`: weighted ce on observation tokens; `score_group`: group-norm credit (inherited) | | `max_rl` | `MaxRLAlgorithm` | `score_group`: mean-normalized group credit | | `rae` | `RAEAlgorithm` | `score_group`: per-agent EMA-baseline credit | -| `hierarchical_grpo` | `HierarchicalGRPOAlgorithm` | `score_group`: two-level peer-set credit (per-episode / per-group) | +| `hierarchical_grpo` | `HierarchicalGRPOAlgorithm` | `score_group`: GRPO baseline per episode for solvers, per group for the proposer | | `opd` | `OPDAlgorithm` | `score_rollout`: own-context prefill under the teacher | | `opsd` | `OPSDAlgorithm` | `score_rollout`: demo-conditioned prefill under the live policy | | `sft` | `SFTDistillAlgorithm` | `score_group`: group-norm credit (feeds filters) | @@ -285,7 +285,7 @@ The per-token training signal is set by `algo.type` and the [algorithm](#the-alg | `grpo` | `rl` | Group-norm: reward minus per-group baseline, optional length penalty. | | `max_rl` | `rl` | Mean-normalized group credit (maximum-likelihood RL). | | `rae` | `rl` | Reward minus a per-agent EMA baseline (SPIRAL's role-conditioned advantage estimation) — for multi-agent self-play envs. | -| `hierarchical_grpo` | `rl` | Two-level peer-set credit for task-generating envs: episode-scoped agents center within their episode, the rest across the group. | +| `hierarchical_grpo` | `rl` | Reward minus a GRPO baseline taken at two levels of the episode tree: each solver against the siblings attempting its own minted task, the proposer across the group's proposals — proposer-solver envs only. | | `echo` | `rl` + `ce` | Group-norm on action tokens, plus weighted CE on env-provided tokens selected by message role (each role's `alpha` is its ECHO λ), optionally narrowed by a user filter. | | `opd` | `ref_kl` | On-policy distillation: per-token reverse KL to a reference model (`teacher`, an inline frozen hosted model), evaluated in the trainer from shipped reference logprobs. No credit — rollouts keep `advantages = None` (advantage-based filters never fire) and ship no advantage stream; `group_size` only fans out sampling. | | `opsd` | `ref_kl` | SDFT: per-token reverse KL to a demo-conditioned reference. No credit — rollouts keep `advantages = None` (advantage-based filters never fire) and ship no advantage stream. | diff --git a/docs/training.md b/docs/training.md index 964ff76a5ca..9327709799d 100644 --- a/docs/training.md +++ b/docs/training.md @@ -90,7 +90,7 @@ The RL entrypoint supports several training algorithms, switched via `[orchestra | `grpo` (default) | None | Standard group-relative RL | | `max_rl` | None | [MaxRL](https://arxiv.org/abs/2602.02710): GRPO with mean-normalized advantages (maximum-likelihood RL) | | `rae` | None | [SPIRAL](https://arxiv.org/abs/2506.24119)'s role-conditioned advantage estimation: reward minus a per-agent EMA baseline, for multi-agent self-play envs (e.g. `kuhn-poker-v1`) | -| `hierarchical_grpo` | None | Two-level GRPO for task-generating envs (e.g. `proposer-solver-v1`): episode-scoped agents baseline within their episode, the rest across the group | +| `hierarchical_grpo` | None | Two-level GRPO for proposer-solver envs: each solver is baselined against the siblings attempting its own minted task, the proposer across the group's proposals | | `opd` | Required, must be vLLM (needs `prompt_logprobs`) | [On-policy distillation](https://thinkingmachines.ai/blog/on-policy-distillation/): the policy generates rollouts, the trainer minimizes per-token reverse KL to a reference model | | `sft` | Required, any OpenAI-compatible endpoint | Hard-distill: a frozen model generates rollouts, the policy trains on its tokens | | `opsd` | None — the live policy is its own reference (no deployment) | [SDFT](https://arxiv.org/abs/2601.19897): the model is its own reference conditioned on expert demonstrations | diff --git a/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py b/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py index ff3841db9fc..7ccf5806ee6 100644 --- a/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py +++ b/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py @@ -397,7 +397,7 @@ def require_frozen_source(self): - ``grpo`` — policy group sampling, group-relative advantage, RL loss (the default). - ``max_rl`` — GRPO with mean-normalized advantages (maximum-likelihood RL). - ``rae`` — reward minus a per-agent EMA baseline (SPIRAL), for multi-agent self-play envs. -- ``hierarchical_grpo`` — two-level GRPO for task-generating envs: episode-scoped agents baseline within their episode, the rest across the group. Needs ``episode_agents``. +- ``hierarchical_grpo`` — two-level GRPO for proposer-solver envs: each solver is baselined against the siblings attempting its own minted task, the proposer across the group's proposals. Needs ``episode_agents``. - ``opd`` — on-policy distillation: policy samples, per-token reverse KL against a reference model. Needs ``teacher``. - ``opsd`` — SDFT: policy samples, demo-conditioned reverse KL against the live policy (the teacher is the policy itself). - ``sft`` — a frozen model samples, the policy trains with CE on its tokens. Needs a frozen ``sampling.source``. From d203da9b12ede5e474a07049c5d451befb167652 Mon Sep 17 00:00:00 2001 From: hallerite Date: Wed, 29 Jul 2026 13:49:25 +0000 Subject: [PATCH 07/10] drop the hierarchical GRPO debug config Co-Authored-By: Claude Opus 5 (1M context) --- configs/debug/algo/hierarchical_grpo.toml | 57 ----------------------- 1 file changed, 57 deletions(-) delete mode 100644 configs/debug/algo/hierarchical_grpo.toml diff --git a/configs/debug/algo/hierarchical_grpo.toml b/configs/debug/algo/hierarchical_grpo.toml deleted file mode 100644 index 0518432c892..00000000000 --- a/configs/debug/algo/hierarchical_grpo.toml +++ /dev/null @@ -1,57 +0,0 @@ -max_steps = 20 -seq_len = 8192 - -[model] -name = "Qwen/Qwen3-4B" - -[wandb] -project = "algorithms-debug" -name = "debug-hierarchical-grpo" - -[orchestrator] -batch_size = 32 -group_size = 4 - -[orchestrator.algo] -type = "hierarchical_grpo" -episode_agents = ["solver"] - -[orchestrator.renderer] -name = "prime-qwen3" - -[orchestrator.train.sampling] -max_completion_tokens = 4096 - -[[orchestrator.train.env]] -name = "proposer-solver" -env.taskset = { id = "proposer-solver-v1" } -env.n = 4 -env.proposer.harness = { id = "null" } -env.proposer.runtime = { type = "subprocess" } -env.solver.harness = { id = "null" } -env.solver.runtime = { type = "subprocess" } - -[orchestrator.eval] -interval = 1 -num_examples = 6 - -[orchestrator.eval.sampling] -temperature = 1.0 -max_completion_tokens = 4096 - -[[orchestrator.eval.env]] -name = "proposer-solver" -env.taskset = { id = "proposer-solver-v1" } -env.n = 4 -env.proposer.harness = { id = "null" } -env.proposer.runtime = { type = "subprocess" } -env.solver.harness = { id = "null" } -env.solver.runtime = { type = "subprocess" } - -[trainer.optim] -lr = 3e-6 - -[ckpt] - -[inference] -gpu_memory_utilization = 0.5 From 22d8cfc0c72a412351892174573fa7b257999bdb Mon Sep 17 00:00:00 2001 From: hallerite Date: Wed, 29 Jul 2026 13:50:37 +0000 Subject: [PATCH 08/10] point the advantage table row at the hierarchical GRPO section Co-Authored-By: Claude Opus 5 (1M context) --- docs/algorithms.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/algorithms.md b/docs/algorithms.md index 630dfc06ff2..a8174dff660 100644 --- a/docs/algorithms.md +++ b/docs/algorithms.md @@ -285,7 +285,7 @@ The per-token training signal is set by `algo.type` and the [algorithm](#the-alg | `grpo` | `rl` | Group-norm: reward minus per-group baseline, optional length penalty. | | `max_rl` | `rl` | Mean-normalized group credit (maximum-likelihood RL). | | `rae` | `rl` | Reward minus a per-agent EMA baseline (SPIRAL's role-conditioned advantage estimation) — for multi-agent self-play envs. | -| `hierarchical_grpo` | `rl` | Reward minus a GRPO baseline taken at two levels of the episode tree: each solver against the siblings attempting its own minted task, the proposer across the group's proposals — proposer-solver envs only. | +| `hierarchical_grpo` | `rl` | For Proposer-Solver type envs. See [Hierarchical Advantage](#hierarchical-advantage-proposersolver). | | `echo` | `rl` + `ce` | Group-norm on action tokens, plus weighted CE on env-provided tokens selected by message role (each role's `alpha` is its ECHO λ), optionally narrowed by a user filter. | | `opd` | `ref_kl` | On-policy distillation: per-token reverse KL to a reference model (`teacher`, an inline frozen hosted model), evaluated in the trainer from shipped reference logprobs. No credit — rollouts keep `advantages = None` (advantage-based filters never fire) and ship no advantage stream; `group_size` only fans out sampling. | | `opsd` | `ref_kl` | SDFT: per-token reverse KL to a demo-conditioned reference. No credit — rollouts keep `advantages = None` (advantage-based filters never fire) and ship no advantage stream. | From fe622fa070918a91fb7af513ef02f0a9f0c452a6 Mon Sep 17 00:00:00 2001 From: hallerite Date: Wed, 29 Jul 2026 13:51:51 +0000 Subject: [PATCH 09/10] move the section pointer up to the algorithm table row Co-Authored-By: Claude Opus 5 (1M context) --- docs/algorithms.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/algorithms.md b/docs/algorithms.md index a8174dff660..eec86a4e6bf 100644 --- a/docs/algorithms.md +++ b/docs/algorithms.md @@ -70,7 +70,7 @@ type = "grpo" # the default | `grpo` | policy | `rl` on actions | Standard group-relative RL. | | `max_rl` | policy | `rl` on actions | MaxRL ([arXiv:2602.02710](https://arxiv.org/abs/2602.02710)): GRPO's centered reward normalized by the group **mean** instead of the standard deviation — the gradient is unbiased for the order-`group_size` truncation of the maximum-likelihood objective, upweighting hard examples like `1/p`. | | `rae` | policy | `rl` on actions | RAE (SPIRAL, [arXiv:2506.24119](https://arxiv.org/abs/2506.24119)): reward minus a per-agent EMA baseline of that agent's own rewards — the estimator for multi-agent self-play envs, where the group mean would mix the agents' opposite reward scales. See [Self-Play Advantage](#self-play-advantage-rae). | -| `hierarchical_grpo` | policy | `rl` on actions | Two-level GRPO for task-generating envs (proposer-solver): agents in `episode_agents` baseline within their own episode (sibling attempts at the same minted task), every other agent across the group (parallel attempts at the same source task). Needs `episode_agents`. See [Hierarchical Advantage](#hierarchical-advantage-proposersolver). | +| `hierarchical_grpo` | policy | `rl` on actions | For Proposer-Solver type envs. Needs `episode_agents`. See [Hierarchical Advantage](#hierarchical-advantage-proposersolver). | | `opd` | policy | `ref_kl` on actions | On-policy distillation ([Thinking Machines](https://thinkingmachines.ai/blog/on-policy-distillation/)): the policy samples, per-token reverse KL against a reference model as the gradient signal. Needs a `teacher`. | | `sft` | *(the teacher)* | `ce` on actions | Hard distillation: a frozen model generates rollouts, the policy trains with CE on its tokens. Needs a frozen `sampling.source` (the teacher it samples from). | | `opsd` | policy | `ref_kl` on actions | SDFT ([arXiv:2601.19897](https://arxiv.org/abs/2601.19897)): the model is its own reference, conditioned on an expert demonstration. The teacher *is* the live policy (the paper's setting, no extra deployment) — no model to configure. | @@ -285,7 +285,7 @@ The per-token training signal is set by `algo.type` and the [algorithm](#the-alg | `grpo` | `rl` | Group-norm: reward minus per-group baseline, optional length penalty. | | `max_rl` | `rl` | Mean-normalized group credit (maximum-likelihood RL). | | `rae` | `rl` | Reward minus a per-agent EMA baseline (SPIRAL's role-conditioned advantage estimation) — for multi-agent self-play envs. | -| `hierarchical_grpo` | `rl` | For Proposer-Solver type envs. See [Hierarchical Advantage](#hierarchical-advantage-proposersolver). | +| `hierarchical_grpo` | `rl` | Reward minus a GRPO baseline taken at two levels of the episode tree: each solver against the siblings attempting its own minted task, the proposer across the group's proposals — proposer-solver envs only. | | `echo` | `rl` + `ce` | Group-norm on action tokens, plus weighted CE on env-provided tokens selected by message role (each role's `alpha` is its ECHO λ), optionally narrowed by a user filter. | | `opd` | `ref_kl` | On-policy distillation: per-token reverse KL to a reference model (`teacher`, an inline frozen hosted model), evaluated in the trainer from shipped reference logprobs. No credit — rollouts keep `advantages = None` (advantage-based filters never fire) and ship no advantage stream; `group_size` only fans out sampling. | | `opsd` | `ref_kl` | SDFT: per-token reverse KL to a demo-conditioned reference. No credit — rollouts keep `advantages = None` (advantage-based filters never fire) and ship no advantage stream. | From 0de29b887b1f73f237c3fbf38ff5fc4ac83db51a Mon Sep 17 00:00:00 2001 From: hallerite Date: Thu, 30 Jul 2026 14:59:52 +0200 Subject: [PATCH 10/10] docs: trim hierarchical GRPO summary --- docs/algorithms.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/algorithms.md b/docs/algorithms.md index dbb967cc1ce..fb6874a58b6 100644 --- a/docs/algorithms.md +++ b/docs/algorithms.md @@ -70,7 +70,7 @@ type = "grpo" # the default | `grpo` | policy | `rl` on actions | Standard group-relative RL. | | `max_rl` | policy | `rl` on actions | MaxRL ([arXiv:2602.02710](https://arxiv.org/abs/2602.02710)): GRPO's centered reward normalized by the group **mean** instead of the standard deviation — the gradient is unbiased for the order-`group_size` truncation of the maximum-likelihood objective, upweighting hard examples like `1/p`. | | `rae` | policy | `rl` on actions | RAE (SPIRAL, [arXiv:2506.24119](https://arxiv.org/abs/2506.24119)): reward minus a per-agent EMA baseline of that agent's own rewards — the estimator for multi-agent self-play envs, where the group mean would mix the agents' opposite reward scales. See [Self-Play Advantage](#self-play-advantage-rae). | -| `hierarchical_grpo` | policy | `rl` on actions | GRPO for proposer-solver envs. Solvers are compared only with attempts on the same proposed problem; proposers are compared with the other proposals in the group. Requires `episode_agents`. See [Hierarchical GRPO](#hierarchical-grpo). | +| `hierarchical_grpo` | policy | `rl` on actions | GRPO for proposer-solver envs. Solvers are compared only with attempts on the same proposed problem; proposers are compared with the other proposals in the group. See [Hierarchical GRPO](#hierarchical-grpo). | | `opd` | policy | `ref_kl` on actions | On-policy distillation ([Thinking Machines](https://thinkingmachines.ai/blog/on-policy-distillation/)): the policy samples, per-token reverse KL against a reference model as the gradient signal. Needs a `teacher`. | | `sft` | *(the teacher)* | `ce` on actions | Hard distillation: a frozen model generates rollouts, the policy trains with CE on its tokens. Needs a frozen `sampling.source` (the teacher it samples from). | | `opsd` | policy | `ref_kl` on actions | SDFT ([arXiv:2601.19897](https://arxiv.org/abs/2601.19897)): the model is its own reference, conditioned on an expert demonstration. The teacher *is* the live policy (the paper's setting, no extra deployment) — no model to configure. |