diff --git a/configs/basic/alphabet-sort/rl.toml b/configs/basic/alphabet-sort/rl.toml index 4b91f67a19..88f537f57b 100644 --- a/configs/basic/alphabet-sort/rl.toml +++ b/configs/basic/alphabet-sort/rl.toml @@ -32,7 +32,7 @@ group_size = 8 [orchestrator.train.sampling] max_completion_tokens = 768 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "alphabet-sort" env.taskset = { id = "alphabet-sort-v1", min_turns = 3, max_turns = 5, task = { power_per_turn = false } } env.agent.harness = { id = "null" } diff --git a/configs/basic/hendrycks-sanity/rl.toml b/configs/basic/hendrycks-sanity/rl.toml index 422720c253..d665e3bd83 100644 --- a/configs/basic/hendrycks-sanity/rl.toml +++ b/configs/basic/hendrycks-sanity/rl.toml @@ -19,7 +19,7 @@ batch_size = 128 group_size = 8 seq_len = 8192 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "hendrycks-math" env.taskset = { id = "math-env-v1", dataset_name = "mikasenghaas/Sanity-Test-R1D-1.5B", dataset_subset = "default" } env.agent.harness = { id = "null" } diff --git a/configs/basic/reverse-text/rl.toml b/configs/basic/reverse-text/rl.toml index 3f7ee602c2..c51c99a405 100644 --- a/configs/basic/reverse-text/rl.toml +++ b/configs/basic/reverse-text/rl.toml @@ -22,7 +22,7 @@ group_size = 16 [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/basic/wiki-search/rl.toml b/configs/basic/wiki-search/rl.toml index c1ec50655f..42a3450c2f 100644 --- a/configs/basic/wiki-search/rl.toml +++ b/configs/basic/wiki-search/rl.toml @@ -40,7 +40,7 @@ group_size = 16 [orchestrator.train.sampling] max_completion_tokens = 512 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "wiki-search" env.taskset = { id = "wiki-search-v1" } env.agent.harness = { id = "null" } diff --git a/configs/basic/wordle/rl.toml b/configs/basic/wordle/rl.toml index 5d46d655eb..31def3ce49 100644 --- a/configs/basic/wordle/rl.toml +++ b/configs/basic/wordle/rl.toml @@ -21,7 +21,7 @@ name = "PrimeIntellect/Qwen3-1.7B-Wordle-SFT" batch_size = 128 group_size = 8 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "wordle" env.taskset = { id = "wordle-v1" } env.player.harness = { id = "null" } diff --git a/configs/ci/integration/alphabet_sort.toml b/configs/ci/integration/alphabet_sort.toml index f63f8636b1..4260790628 100644 --- a/configs/ci/integration/alphabet_sort.toml +++ b/configs/ci/integration/alphabet_sort.toml @@ -22,7 +22,7 @@ group_size = 8 [orchestrator.train.sampling] max_completion_tokens = 384 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "alphabet-sort" env.taskset = { id = "alphabet-sort-v1", min_turns = 2, max_turns = 2, min_names_per_turn = 1, max_names_per_turn = 3, task = { similarity_power = 4, power_per_turn = false } } env.agent.harness = { id = "null" } diff --git a/configs/ci/integration/reverse-text-lora/resume.toml b/configs/ci/integration/reverse-text-lora/resume.toml index 372cfc4c9a..d91fada025 100644 --- a/configs/ci/integration/reverse-text-lora/resume.toml +++ b/configs/ci/integration/reverse-text-lora/resume.toml @@ -26,7 +26,7 @@ name = "r8-1e-4" [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/ci/integration/reverse-text-lora/start.toml b/configs/ci/integration/reverse-text-lora/start.toml index e3b867d7dd..4c5428af5e 100644 --- a/configs/ci/integration/reverse-text-lora/start.toml +++ b/configs/ci/integration/reverse-text-lora/start.toml @@ -25,7 +25,7 @@ name = "r8-1e-4" [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/ci/integration/reverse-text-moe/start.toml b/configs/ci/integration/reverse-text-moe/start.toml index e4a0093d60..8a6ee084f3 100644 --- a/configs/ci/integration/reverse-text-moe/start.toml +++ b/configs/ci/integration/reverse-text-moe/start.toml @@ -25,7 +25,7 @@ group_size = 16 [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/ci/integration/reverse-text-multi-run/orchestrator.toml b/configs/ci/integration/reverse-text-multi-run/orchestrator.toml index dab5fe8cdd..1a1c29eb75 100644 --- a/configs/ci/integration/reverse-text-multi-run/orchestrator.toml +++ b/configs/ci/integration/reverse-text-multi-run/orchestrator.toml @@ -14,7 +14,7 @@ lr = 3e-5 [train.sampling] max_completion_tokens = 128 -[[train.env]] +[[train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/ci/integration/reverse-text-rl-opd/start.toml b/configs/ci/integration/reverse-text-rl-opd/start.toml index 2069940008..2092ee5cb3 100644 --- a/configs/ci/integration/reverse-text-rl-opd/start.toml +++ b/configs/ci/integration/reverse-text-rl-opd/start.toml @@ -37,7 +37,7 @@ name = "prime-qwen3" [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } @@ -50,7 +50,7 @@ num_examples = 16 [orchestrator.eval.sampling] max_completion_tokens = 128 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/ci/integration/reverse-text-rl-sft/start.toml b/configs/ci/integration/reverse-text-rl-sft/start.toml index c8b3d08a8b..be5fc99c67 100644 --- a/configs/ci/integration/reverse-text-rl-sft/start.toml +++ b/configs/ci/integration/reverse-text-rl-sft/start.toml @@ -43,7 +43,7 @@ name = "prime-qwen3" [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } @@ -56,7 +56,7 @@ num_examples = 16 [orchestrator.eval.sampling] max_completion_tokens = 128 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/ci/integration/reverse-text/resume.toml b/configs/ci/integration/reverse-text/resume.toml index 06fab44ac5..7d351d99fd 100644 --- a/configs/ci/integration/reverse-text/resume.toml +++ b/configs/ci/integration/reverse-text/resume.toml @@ -23,7 +23,7 @@ group_size = 16 [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/ci/integration/reverse-text/start.toml b/configs/ci/integration/reverse-text/start.toml index 11facde4c8..1399c176d6 100644 --- a/configs/ci/integration/reverse-text/start.toml +++ b/configs/ci/integration/reverse-text/start.toml @@ -22,7 +22,7 @@ group_size = 16 [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/ci/nightly-fft/alphabet-sort.toml b/configs/ci/nightly-fft/alphabet-sort.toml index 72e4ef453a..2d54860f1d 100644 --- a/configs/ci/nightly-fft/alphabet-sort.toml +++ b/configs/ci/nightly-fft/alphabet-sort.toml @@ -18,7 +18,7 @@ name = "Qwen/Qwen3-4B-Instruct-2507" batch_size = 512 group_size = 16 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "alphabet-sort" env.taskset = { id = "alphabet-sort-v1", min_turns = 3, max_turns = 5, task = { power_per_turn = false } } env.agent.harness = { id = "null" } diff --git a/configs/ci/nightly-fft/hendrycks-sanity.toml b/configs/ci/nightly-fft/hendrycks-sanity.toml index 575ee690ed..be52cca7d4 100644 --- a/configs/ci/nightly-fft/hendrycks-sanity.toml +++ b/configs/ci/nightly-fft/hendrycks-sanity.toml @@ -18,7 +18,7 @@ batch_size = 512 group_size = 8 seq_len = 8192 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "hendrycks-math" env.taskset = { id = "math-env-v1", dataset_name = "mikasenghaas/Sanity-Test-R1D-1.5B", dataset_subset = "default" } env.agent.harness = { id = "null" } @@ -27,7 +27,7 @@ env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 50 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "aime2024" env.taskset = { id = "aime24-v1" } env.agent.harness = { id = "null" } diff --git a/configs/ci/nightly-fft/reverse-text.toml b/configs/ci/nightly-fft/reverse-text.toml index 73d95d0050..278307ed2d 100644 --- a/configs/ci/nightly-fft/reverse-text.toml +++ b/configs/ci/nightly-fft/reverse-text.toml @@ -18,7 +18,7 @@ name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT" batch_size = 128 group_size = 16 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/ci/nightly-fft/wiki-search.toml b/configs/ci/nightly-fft/wiki-search.toml index 8b90fdae61..d625fb6d76 100644 --- a/configs/ci/nightly-fft/wiki-search.toml +++ b/configs/ci/nightly-fft/wiki-search.toml @@ -23,7 +23,7 @@ oversampling_factor = 2.0 type = "zero_advantage" enforce = true -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "wiki-search" env.taskset = { id = "wiki-search-v1" } env.agent.harness = { id = "null" } diff --git a/configs/ci/nightly-fft/wordle.toml b/configs/ci/nightly-fft/wordle.toml index ad10225aeb..27da7f7c84 100644 --- a/configs/ci/nightly-fft/wordle.toml +++ b/configs/ci/nightly-fft/wordle.toml @@ -18,7 +18,7 @@ name = "PrimeIntellect/Qwen3-1.7B-Wordle-SFT" batch_size = 512 group_size = 16 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "wordle" env.taskset = { id = "wordle-v1" } env.player.harness = { id = "null" } diff --git a/configs/ci/nightly/multimodal_color_codeword.toml b/configs/ci/nightly/multimodal_color_codeword.toml index 5ce4971998..42997813f8 100644 --- a/configs/ci/nightly/multimodal_color_codeword.toml +++ b/configs/ci/nightly/multimodal_color_codeword.toml @@ -20,7 +20,7 @@ group_size = 16 [orchestrator.train.sampling] max_completion_tokens = 64 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "color-codeword" env.taskset = { id = "color-codeword-v1", images_per_turn = 1 } env.agent.harness = { id = "null" } diff --git a/configs/debug/algo/echo.toml b/configs/debug/algo/echo.toml index 0805249c2b..c4a50764d5 100644 --- a/configs/debug/algo/echo.toml +++ b/configs/debug/algo/echo.toml @@ -24,7 +24,7 @@ type = "echo" [orchestrator.algo.roles.user] alpha = 0.1 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "alphabet-sort" env.taskset = { id = "alphabet-sort-v1", min_turns = 3, max_turns = 5, task = { power_per_turn = false } } env.agent.harness = { id = "null" } diff --git a/configs/debug/algo/grpo.toml b/configs/debug/algo/grpo.toml index 25da69ec8d..2ea6c5e762 100644 --- a/configs/debug/algo/grpo.toml +++ b/configs/debug/algo/grpo.toml @@ -21,7 +21,7 @@ name = "prime-qwen3" [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } @@ -34,7 +34,7 @@ num_examples = 128 [orchestrator.eval.sampling] max_completion_tokens = 128 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/debug/algo/max_rl.toml b/configs/debug/algo/max_rl.toml index 620fbd2b98..77837e60d2 100644 --- a/configs/debug/algo/max_rl.toml +++ b/configs/debug/algo/max_rl.toml @@ -21,7 +21,7 @@ name = "prime-qwen3" [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } @@ -34,7 +34,7 @@ num_examples = 128 [orchestrator.eval.sampling] max_completion_tokens = 128 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/debug/algo/mixed_grpo_opd.toml b/configs/debug/algo/mixed_grpo_opd.toml index a4a24ec4b4..3cd80ea6a9 100644 --- a/configs/debug/algo/mixed_grpo_opd.toml +++ b/configs/debug/algo/mixed_grpo_opd.toml @@ -31,22 +31,22 @@ name = "prime-qwen3" [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text-grpo" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } env.agent.runtime = { type = "subprocess" } -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text-opd" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } env.agent.runtime = { type = "subprocess" } -[orchestrator.train.env.algo] +[orchestrator.train.source.algo] type = "opd" -[orchestrator.train.env.algo.teacher] +[orchestrator.train.source.algo.teacher] name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL" base_url = ["http://localhost:8001/v1"] @@ -57,7 +57,7 @@ num_examples = 128 [orchestrator.eval.sampling] max_completion_tokens = 128 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/debug/algo/opd.toml b/configs/debug/algo/opd.toml index 13d2950c73..8dbdf93d8c 100644 --- a/configs/debug/algo/opd.toml +++ b/configs/debug/algo/opd.toml @@ -33,7 +33,7 @@ name = "prime-qwen3" [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } @@ -46,7 +46,7 @@ num_examples = 128 [orchestrator.eval.sampling] max_completion_tokens = 128 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/debug/algo/opd_lora.toml b/configs/debug/algo/opd_lora.toml index a6f0eb9550..147f0be9d5 100644 --- a/configs/debug/algo/opd_lora.toml +++ b/configs/debug/algo/opd_lora.toml @@ -33,7 +33,7 @@ name = "prime-qwen3" [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } @@ -46,7 +46,7 @@ num_examples = 128 [orchestrator.eval.sampling] max_completion_tokens = 128 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/debug/algo/rae.toml b/configs/debug/algo/rae.toml index e601277aa9..67823dc4a2 100644 --- a/configs/debug/algo/rae.toml +++ b/configs/debug/algo/rae.toml @@ -21,7 +21,7 @@ name = "prime-qwen3" [orchestrator.train.sampling] max_completion_tokens = 512 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "kuhn-poker" env.taskset = { id = "kuhn-poker-v1" } env.player0.harness = { id = "null" } @@ -36,7 +36,7 @@ num_examples = 16 [orchestrator.eval.sampling] max_completion_tokens = 512 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "kuhn-poker" env.taskset = { id = "kuhn-poker-v1" } env.player0.harness = { id = "null" } diff --git a/configs/debug/algo/self_distill.toml b/configs/debug/algo/self_distill.toml index 93959f7b22..1a3fbbcc1d 100644 --- a/configs/debug/algo/self_distill.toml +++ b/configs/debug/algo/self_distill.toml @@ -29,7 +29,7 @@ name = "prime-qwen3" [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } @@ -42,7 +42,7 @@ num_examples = 128 [orchestrator.eval.sampling] max_completion_tokens = 128 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/debug/algo/sft_distill.toml b/configs/debug/algo/sft_distill.toml index dbc0986f2c..521bcce29b 100644 --- a/configs/debug/algo/sft_distill.toml +++ b/configs/debug/algo/sft_distill.toml @@ -38,7 +38,7 @@ name = "prime-qwen3" [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } @@ -51,7 +51,7 @@ num_examples = 128 [orchestrator.eval.sampling] max_completion_tokens = 128 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/configs/debug/algo/sft_distill_lora.toml b/configs/debug/algo/sft_distill_lora.toml index 51f1034c6a..9494a67f03 100644 --- a/configs/debug/algo/sft_distill_lora.toml +++ b/configs/debug/algo/sft_distill_lora.toml @@ -38,7 +38,7 @@ name = "prime-qwen3" [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } @@ -51,7 +51,7 @@ num_examples = 128 [orchestrator.eval.sampling] max_completion_tokens = 128 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/deps/verifiers b/deps/verifiers index d0bb0ffe53..fcb48822ea 160000 --- a/deps/verifiers +++ b/deps/verifiers @@ -1 +1 @@ -Subproject commit d0bb0ffe5326ff9cfff9513c0e046bd0977ac1e0 +Subproject commit fcb48822eaf35efe22b6f6deda9c4b422920314e diff --git a/docs/algorithms.md b/docs/algorithms.md index 48a599f251..2c8846e09f 100644 --- a/docs/algorithms.md +++ b/docs/algorithms.md @@ -118,18 +118,18 @@ Both components resolve per environment. Each env inherits `[orchestrator.algo]` [orchestrator.algo] type = "grpo" -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "math" -taskset = { id = "math-v1" } -harness = { id = "null" } -runtime = { type = "subprocess" } +env.taskset = { id = "math-v1" } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } # inherits the top-level grpo -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "terminal" -taskset = { id = "terminal-v1" } -harness = { id = "bash" } -runtime = { type = "subprocess" } +env.taskset = { id = "terminal-v1" } +env.agent.harness = { id = "bash" } +env.agent.runtime = { type = "subprocess" } algo = { type = "echo" } # this env runs its own algorithm ``` @@ -314,7 +314,7 @@ Group-relative baselines assume the group is exchangeable attempts by one agent. type = "rae" decay = 0.95 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "kuhn-poker" env.taskset = { id = "kuhn-poker-v1" } env.player0.harness = { id = "null" } diff --git a/docs/configuration.md b/docs/configuration.md index 4b0474edeb..a6d75e7118 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -17,7 +17,7 @@ Every `prime-rl` entrypoint uses [`pydantic-config`](https://github.com/PrimeInt - [Optional Sub-Configs](#optional-sub-configs) - [None](#none) - [Discriminated Unions](#discriminated-unions) - - [Environments](#environments-orchestratortrainenv) + - [Environments](#environments) - [Environment Variables](#environment-variables) - [Examples](#examples) @@ -93,19 +93,19 @@ uv run rl @ rl.toml --trainer.model.lora.target-modules '["q_proj", "k_proj", "v target_modules = ["q_proj", "k_proj", "v_proj"] ``` -Overlay TOMLs **replace** lists wholesale — an overlay that wants to add one item must still spell out the full list. For arrays of tables (e.g. environments), see [Environments](#environments-orchestratortrainenv). +Overlay TOMLs **replace** lists wholesale — an overlay that wants to add one item must still spell out the full list. For arrays of tables, see [Environments](#environments). ### Dicts CLI takes a JSON literal. TOML uses a table or inline-table. CLI dicts deep-merge with TOML dicts — CLI keys win on conflict but don't wipe the file's keys: ```bash -uv run rl @ rl.toml --orchestrator.train.env.0.args \ +uv run rl @ rl.toml --orchestrator.train.source.0.args \ '{"dataset_name": "openai/gsm8k", "dataset_subset": "main"}' ``` ```toml -[[orchestrator.train.env]] +[[orchestrator.train.source]] args = { dataset_name = "openai/gsm8k", dataset_subset = "main" } ``` @@ -142,26 +142,26 @@ mu = 0.95 Omit `type` to keep the default variant. -### Environments (`[[orchestrator.train.env]]`) +### Environments -Training environments are an array of tables — set one per env, optionally with sampling weights: +Training and evaluation sources are arrays of tables. Set one source per environment; training sources can optionally carry sampling weights: ```toml -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "gsm8k" env.taskset = { id = "gsm8k-v1", split = "train" } env.agent.harness = { id = "null" } env.agent.runtime = { type = "subprocess" } ratio = 3 # 75% of batches -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } env.agent.runtime = { type = "subprocess" } ratio = 1 # default — 25% of batches -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "gsm8k-eval" env.taskset = { id = "gsm8k-v1", split = "test" } env.agent.harness = { id = "null" } diff --git a/docs/inference.md b/docs/inference.md index fcf82e72d9..a99b67b117 100644 --- a/docs/inference.md +++ b/docs/inference.md @@ -279,6 +279,6 @@ enable_router_replay = true # this will also auto-set the inference.enable_retur enable_return_routed_experts = true ``` -This however is not free, it adds a significant overhead to the HTTP requests as this payload can grow quite large. We reccomend increasing `orchestrator.*.env.num_workers` to allow for more parallelization on the verifiers side. +This however is not free, it adds a significant overhead to the HTTP requests as this payload can grow quite large. We reccomend sizing up the env-server pool (`orchestrator.*.source.serve.pool`) to allow for more parallelization on the verifiers side. Currently this feature is also not supported with CPU KV cache offload, which can have negative impact on the inference throughput. diff --git a/docs/training.md b/docs/training.md index 12be2dc58f..d6926e84ce 100644 --- a/docs/training.md +++ b/docs/training.md @@ -60,8 +60,8 @@ A condensed view of the knobs you'll most often tune. For trainer-side paralleli | `orchestrator.group_size` | Rollouts generated per task. | | `orchestrator.max_off_policy_steps` | How many distinct policies may have contributed to one rollout before it's discarded (default 8). The main off-policy dial on long agentic rollouts — bump for throughput, lower for tighter on-policyness. Watch `errored_rollouts` and `mismatch_kl/all/mean` when tuning. | | `[orchestrator.algo]` | Training algorithm — its `type` names it (`grpo` default, `max_rl`, `rae`, `opd`, `opsd`, `sft`, `echo`). See [Algorithms](#algorithms). | -| `[[orchestrator.train.env]]` | Training environments. List multiple tables for multi-env training; weight them via `ratio`. See [Configuration § Environments](configuration.md#environments-orchestratortrainenv). | -| `[[orchestrator.eval.env]]` + `orchestrator.eval.interval` | Eval environments and cadence (default every 100 steps). | +| `[[orchestrator.train.source]]` | Training sources. List multiple tables for multi-env training; weight them via `ratio`. See [Configuration § Training sources](configuration.md#training-sources-orchestratortrainsource). | +| `[[orchestrator.eval.source]]` + `orchestrator.eval.interval` | Eval environments and cadence (default every 100 steps). | **Monitoring:** diff --git a/examples/advanced/glm-4.5-air/search.toml b/examples/advanced/glm-4.5-air/search.toml index e2d3dfc997..402d392b5f 100644 --- a/examples/advanced/glm-4.5-air/search.toml +++ b/examples/advanced/glm-4.5-air/search.toml @@ -71,7 +71,7 @@ type = "muon" batch_size = 256 group_size = 16 max_off_policy_steps = 32 -max_inflight_rollouts = 512 +max_inflight_episodes = 512 [orchestrator.renderer] name = "glm-4.5" @@ -93,15 +93,15 @@ num_turns_weight = 0.0 # openseeker/redsearcher pre-plug a `reference` judge; declaring `judges` here replaces that # entry wholesale (keep `name`/`question_field`/`choices` in sync with the taskset defaults — # the plugged BC-style prompt is replaced by the reference judge's stock prompt). -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "openseeker" env.agent.harness = { id = "rlm", skills = ["search"], forward_env = ["SERPER_API_KEY"], summarize_at_tokens = [65536, 131072] } env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-search"] } -[orchestrator.train.env.env.taskset] +[orchestrator.train.source.env.taskset] id = "openseeker-v1" -[[orchestrator.train.env.env.taskset.task.judges]] +[[orchestrator.train.source.env.taskset.task.judges]] id = "reference" name = "correct" model = "Qwen/Qwen3-235B-A22B-Instruct-2507" @@ -109,22 +109,22 @@ question_field = "question" choices = ["A", "B"] # # Metric-only monitor (weight 0): rlm harness-mechanics rubric, judged by GLM-5.2. -# [[orchestrator.train.env.env.taskset.task.judges]] +# [[orchestrator.train.source.env.taskset.task.judges]] # id = "rubric" # name = "rlm" # path = "deps/research-environments/rubrics/rlm.toml" # model = "zai-org/GLM-5.2" # weight = 0.0 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "redsearcher" env.agent.harness = { id = "rlm", skills = ["search"], forward_env = ["SERPER_API_KEY"], summarize_at_tokens = [65536, 131072] } env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-search"] } -[orchestrator.train.env.env.taskset] +[orchestrator.train.source.env.taskset] id = "redsearcher-v1" # optional `difficulty` filter (easy/medium/hard) -[[orchestrator.train.env.env.taskset.task.judges]] +[[orchestrator.train.source.env.taskset.task.judges]] id = "reference" name = "correct" model = "Qwen/Qwen3-235B-A22B-Instruct-2507" @@ -132,7 +132,7 @@ question_field = "question" choices = ["A", "B"] # # Metric-only monitor (weight 0): rlm harness-mechanics rubric, judged by GLM-5.2. -# [[orchestrator.train.env.env.taskset.task.judges]] +# [[orchestrator.train.source.env.taskset.task.judges]] # id = "rubric" # name = "rlm" # path = "deps/research-environments/rubrics/rlm.toml" @@ -144,19 +144,19 @@ choices = ["A", "B"] [orchestrator.eval] interval = 20 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "browsecomp" num_examples = 500 # full set is 1266; cap eval at 500 env.agent.harness = { id = "rlm", skills = ["search"], forward_env = ["SERPER_API_KEY"], summarize_at_tokens = 98304 } env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-search"] } env.agent.timeout = { rollout = 3600 } -[orchestrator.eval.env.env.taskset] +[orchestrator.eval.source.env.taskset] id = "browsecomp-v1" task = { judge = { model = "Qwen/Qwen3-235B-A22B-Instruct-2507" } } # # Metric-only monitor (weight 0): rlm harness-mechanics rubric, judged by GLM-5.2. -# [[orchestrator.eval.env.env.taskset.task.judges]] +# [[orchestrator.eval.source.env.taskset.task.judges]] # id = "rubric" # name = "rlm" # path = "deps/research-environments/rubrics/rlm.toml" diff --git a/examples/advanced/glm-4.5-air/swe.toml b/examples/advanced/glm-4.5-air/swe.toml index b9de4512cc..b712ac1d19 100644 --- a/examples/advanced/glm-4.5-air/swe.toml +++ b/examples/advanced/glm-4.5-air/swe.toml @@ -82,7 +82,7 @@ type = "muon" batch_size = 256 group_size = 16 max_off_policy_steps = 32 -max_inflight_rollouts = 512 +max_inflight_episodes = 512 [orchestrator.algo] type = "grpo" @@ -97,7 +97,7 @@ num_turns_weight = 0.1 name = "glm-4.5" thinking_retention = "all" -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "scaleswe" env.taskset = { id = "scaleswe-v1" } env.agent.harness = { id = "rlm", summarize_at_tokens = [65536, 131072] } @@ -110,7 +110,7 @@ env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = [orchestrator.eval] interval = 20 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "swebench-verified" env.taskset = { id = "swebench-verified-v1" } env.agent.harness = { id = "rlm", summarize_at_tokens = 98304 } diff --git a/examples/advanced/glm-4.5-air/terminal.toml b/examples/advanced/glm-4.5-air/terminal.toml index 7dfd984e1a..42e0fe77f5 100644 --- a/examples/advanced/glm-4.5-air/terminal.toml +++ b/examples/advanced/glm-4.5-air/terminal.toml @@ -72,7 +72,7 @@ type = "muon" batch_size = 256 group_size = 16 max_off_policy_steps = 32 -max_inflight_rollouts = 512 +max_inflight_episodes = 512 [orchestrator.renderer] name = "glm-4.5" @@ -91,23 +91,23 @@ num_turns_weight = 0.1 # --- Train: tmax (per-task Docker image in a prime sandbox, rlm ipython) --- -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "tmax" env.agent.harness = { id = "rlm", summarize_at_tokens = [65536, 131072] } env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-terminal"] } -[orchestrator.train.env.env.taskset] +[orchestrator.train.source.env.taskset] id = "tmax-v1" # # Metric-only monitors (weight 0): rlm harness-mechanics + swe code-usage rubrics, judged by GLM-5.2. -# [[orchestrator.train.env.env.taskset.judges]] +# [[orchestrator.train.source.env.taskset.judges]] # id = "rubric" # name = "rlm" # path = "deps/research-environments/rubrics/rlm.toml" # model = "zai-org/GLM-5.2" # weight = 0.0 -# [[orchestrator.train.env.env.taskset.judges]] +# [[orchestrator.train.source.env.taskset.judges]] # id = "rubric" # name = "swe" # path = "deps/research-environments/rubrics/swe.toml" @@ -119,47 +119,47 @@ id = "tmax-v1" [orchestrator.eval] interval = 20 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "swebench-verified" env.agent.harness = { id = "rlm", summarize_at_tokens = 98304 } env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-terminal"] } env.agent.timeout = { rollout = 3600 } -[orchestrator.eval.env.env.taskset] +[orchestrator.eval.source.env.taskset] id = "swebench-verified-v1" -# [[orchestrator.eval.env.env.taskset.judges]] +# [[orchestrator.eval.source.env.taskset.judges]] # id = "rubric" # name = "rlm" # path = "deps/research-environments/rubrics/rlm.toml" # model = "zai-org/GLM-5.2" # weight = 0.0 -# [[orchestrator.eval.env.env.taskset.judges]] +# [[orchestrator.eval.source.env.taskset.judges]] # id = "rubric" # name = "swe" # path = "deps/research-environments/rubrics/swe.toml" # model = "zai-org/GLM-5.2" # weight = 0.0 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "terminal-bench-2" group_size = 4 # avg@4 env.agent.harness = { id = "rlm", summarize_at_tokens = 98304 } env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = ["glm45air-terminal"] } env.agent.timeout = { rollout = 3600 } -[orchestrator.eval.env.env.taskset] +[orchestrator.eval.source.env.taskset] id = "terminal-bench-2-v1" -# [[orchestrator.eval.env.env.taskset.judges]] +# [[orchestrator.eval.source.env.taskset.judges]] # id = "rubric" # name = "rlm" # path = "deps/research-environments/rubrics/rlm.toml" # model = "zai-org/GLM-5.2" # weight = 0.0 -# [[orchestrator.eval.env.env.taskset.judges]] +# [[orchestrator.eval.source.env.taskset.judges]] # id = "rubric" # name = "swe" # path = "deps/research-environments/rubrics/swe.toml" diff --git a/examples/advanced/glm-5.2/swe-llmd.toml b/examples/advanced/glm-5.2/swe-llmd.toml index 4fdb589961..3b5562bf10 100644 --- a/examples/advanced/glm-5.2/swe-llmd.toml +++ b/examples/advanced/glm-5.2/swe-llmd.toml @@ -81,7 +81,7 @@ weight_decay = 0.0 [orchestrator] batch_size = 256 group_size = 16 -max_inflight_rollouts = 2048 +max_inflight_episodes = 2048 max_off_policy_steps = 8 # 16/32 can be better, 8 is more CPU stable [orchestrator.model] @@ -96,7 +96,7 @@ temperature = 1.0 extra_body = {chat_template_kwargs = {clear_thinking = false}} -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "r2e" env.taskset = { id = "r2e-gym-v1" } env.agent.harness = { id = "rlm" } diff --git a/examples/advanced/glm-5.2/swe.toml b/examples/advanced/glm-5.2/swe.toml index 99d2b31f70..4a565dd6b8 100644 --- a/examples/advanced/glm-5.2/swe.toml +++ b/examples/advanced/glm-5.2/swe.toml @@ -71,7 +71,7 @@ name = "zai-org/GLM-5-FP8" [orchestrator.eval] interval = 20 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "swe-bench-verified" env.taskset = { id = "swebench-verified-v1" } env.agent.harness = { id = "bash" } diff --git a/examples/advanced/intellect-3.1/rl.toml b/examples/advanced/intellect-3.1/rl.toml index a85ba04cdc..dae0c57691 100644 --- a/examples/advanced/intellect-3.1/rl.toml +++ b/examples/advanced/intellect-3.1/rl.toml @@ -47,35 +47,35 @@ weight_decay = 0.01 batch_size = 2048 oversampling_factor = 2 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "swe" ratio = 0.3 env.taskset = { id = "r2e-gym-v1" } env.agent.harness = { id = "bash" } env.agent.runtime = { type = "prime", labels = ["intellect-3.1", "swe"] } -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "deepdive" ratio = 0.2 env.taskset = { id = "deepdive-v1" } env.agent.harness = { id = "rlm", skills = ["search"], forward_env = ["SERPER_API_KEY"] } env.agent.runtime = { type = "prime", labels = ["intellect-3.1", "deepdive"] } -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "math" ratio = 0.3 env.taskset = { id = "i3_math_v1" } env.agent.harness = { id = "null" } env.agent.runtime = { type = "subprocess" } -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "logic" ratio = 0.2 env.taskset = { id = "i3_logic_v1", filter = { max = 0.874 } } env.agent.harness = { id = "null" } env.agent.runtime = { type = "subprocess" } -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "code" ratio = 0.2 env.taskset = { id = "i3_code_v1" } @@ -89,13 +89,13 @@ enforce = true [orchestrator.eval] interval = 25 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "swe-bench-verified" env.taskset = { id = "swebench-verified-v1" } env.agent.harness = { id = "bash" } env.agent.runtime = { type = "prime", labels = ["intellect-3.1", "swe-bench-verified"] } -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "aime2025" env.taskset = { id = "aime25-v1" } env.agent.harness = { id = "null" } diff --git a/examples/advanced/minimax-m2.5/swe.toml b/examples/advanced/minimax-m2.5/swe.toml index 76035905c4..55f7e9efc4 100644 --- a/examples/advanced/minimax-m2.5/swe.toml +++ b/examples/advanced/minimax-m2.5/swe.toml @@ -49,7 +49,7 @@ oversampling_factor = 2 max_off_policy_steps = 16 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "swe" env.taskset = { id = "r2e-gym-v1" } env.agent.harness = { id = "bash" } @@ -58,7 +58,7 @@ env.agent.runtime = { type = "prime", labels = ["minimax-swe", "swe"] } [orchestrator.eval] interval = 25 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "swe-bench-verified" env.taskset = { id = "swebench-verified-v1" } env.agent.harness = { id = "bash" } diff --git a/examples/advanced/nemotron-3-super/swe.toml b/examples/advanced/nemotron-3-super/swe.toml index 139ff77e04..f96c292bcb 100644 --- a/examples/advanced/nemotron-3-super/swe.toml +++ b/examples/advanced/nemotron-3-super/swe.toml @@ -66,7 +66,7 @@ type = "adamw" batch_size = 256 group_size = 16 max_off_policy_steps = 32 -max_inflight_rollouts = 1536 +max_inflight_episodes = 1536 [orchestrator.algo] type = "grpo" @@ -81,7 +81,7 @@ num_turns_weight = 0.1 name = "nemotron-3" truncate_history_thinking = false -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "scaleswe" env.taskset = { id = "scaleswe-v1" } env.agent.harness = { id = "rlm", summarize_at_tokens = [65536, 131072] } @@ -92,7 +92,7 @@ env.agent.runtime = { type = "prime", vm = true, idle_timeout = "None", labels = [orchestrator.eval] interval = 20 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "swebench-verified" env.taskset = { id = "swebench-verified-v1" } env.agent.harness = { id = "rlm", summarize_at_tokens = 98304 } diff --git a/examples/advanced/qwen3-30b-a3b/math.toml b/examples/advanced/qwen3-30b-a3b/math.toml index 4b9f326da1..0691bd26a2 100644 --- a/examples/advanced/qwen3-30b-a3b/math.toml +++ b/examples/advanced/qwen3-30b-a3b/math.toml @@ -48,7 +48,7 @@ max_off_policy_steps = 8 [orchestrator.train.sampling] max_completion_tokens = 32768 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "math" env.taskset = { id = "i3_math_v1" } env.agent.harness = { id = "null" } @@ -57,7 +57,7 @@ env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 25 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "aime2025" env.taskset = { id = "aime25-v1" } env.agent.harness = { id = "null" } diff --git a/examples/advanced/qwen3-30b-a3b/swe.toml b/examples/advanced/qwen3-30b-a3b/swe.toml index 06b50eeba9..0d7271e2b7 100644 --- a/examples/advanced/qwen3-30b-a3b/swe.toml +++ b/examples/advanced/qwen3-30b-a3b/swe.toml @@ -46,7 +46,7 @@ batch_size = 512 oversampling_factor = 2 max_off_policy_steps = 16 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "swe" env.taskset = { id = "r2e-gym-v1" } env.agent.harness = { id = "bash" } @@ -55,7 +55,7 @@ env.agent.runtime = { type = "prime", labels = ["qwen30b-swe", "swe"] } [orchestrator.eval] interval = 25 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "swe-bench-verified" env.taskset = { id = "swebench-verified-v1" } env.agent.harness = { id = "bash" } diff --git a/examples/advanced/qwen3-30b-a3b/tool.toml b/examples/advanced/qwen3-30b-a3b/tool.toml index 375d66afd6..892f106e49 100644 --- a/examples/advanced/qwen3-30b-a3b/tool.toml +++ b/examples/advanced/qwen3-30b-a3b/tool.toml @@ -35,7 +35,7 @@ batch_size = 512 group_size = 16 max_off_policy_steps = 32 -[[orchestrator.train.env]] +[[orchestrator.train.source]] env.taskset = { id = "general-agent-v1", task = { tools = { colocated = true } }, min_tier = 1 } env.agent.harness = { id = "null" } env.agent.runtime = { type = "modal" } diff --git a/examples/basic/alphabet-sort/rl.toml b/examples/basic/alphabet-sort/rl.toml index 252de1bc93..f1c691b0ac 100644 --- a/examples/basic/alphabet-sort/rl.toml +++ b/examples/basic/alphabet-sort/rl.toml @@ -31,7 +31,7 @@ group_size = 16 [orchestrator.train.sampling] max_completion_tokens = 768 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "alphabet-sort" env.taskset = { id = "alphabet-sort-v1", min_turns = 3, max_turns = 5, task = { power_per_turn = false } } env.agent.harness = { id = "null" } diff --git a/examples/basic/hendrycks-sanity/rl.toml b/examples/basic/hendrycks-sanity/rl.toml index 40812973a1..80d6583c78 100644 --- a/examples/basic/hendrycks-sanity/rl.toml +++ b/examples/basic/hendrycks-sanity/rl.toml @@ -16,7 +16,7 @@ batch_size = 512 group_size = 8 seq_len = 8192 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "hendrycks-math" env.taskset = { id = "math-env-v1", dataset_name = "mikasenghaas/Sanity-Test-R1D-1.5B", dataset_subset = "default" } env.agent.harness = { id = "null" } @@ -25,7 +25,7 @@ env.agent.runtime = { type = "subprocess" } [orchestrator.eval] interval = 50 -[[orchestrator.eval.env]] +[[orchestrator.eval.source]] name = "aime2024" env.taskset = { id = "aime24-v1" } env.agent.harness = { id = "null" } diff --git a/examples/basic/reverse-text/rl.toml b/examples/basic/reverse-text/rl.toml index 08990f52a4..6dfe31fb5d 100644 --- a/examples/basic/reverse-text/rl.toml +++ b/examples/basic/reverse-text/rl.toml @@ -15,7 +15,7 @@ group_size = 16 [orchestrator.train.sampling] max_completion_tokens = 128 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } diff --git a/examples/basic/wiki-search/README.md b/examples/basic/wiki-search/README.md index a69009448c..a04ad46aba 100644 --- a/examples/basic/wiki-search/README.md +++ b/examples/basic/wiki-search/README.md @@ -133,7 +133,7 @@ uv run eval wiki-search-v1 --harness.id null \ The V1 taskset fixes the question bank and searchable corpus. You can replace its reference judge in `rl.toml`: ```toml -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "wiki-search" taskset = { id = "wiki-search-v1", task = { judges = [{ id = "reference", model = "openai/gpt-5.4-nano" }] } } harness = { id = "null" } diff --git a/examples/basic/wiki-search/rl.toml b/examples/basic/wiki-search/rl.toml index f88913036c..4e0bb97388 100644 --- a/examples/basic/wiki-search/rl.toml +++ b/examples/basic/wiki-search/rl.toml @@ -44,7 +44,7 @@ max_completion_tokens = 512 type = "zero_advantage" enforce = true -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "wiki-search" env.taskset = { id = "wiki-search-v1" } env.agent.harness = { id = "null" } diff --git a/examples/basic/wordle/rl.toml b/examples/basic/wordle/rl.toml index 5e40e82629..4ecb5611ab 100644 --- a/examples/basic/wordle/rl.toml +++ b/examples/basic/wordle/rl.toml @@ -18,7 +18,7 @@ name = "PrimeIntellect/Qwen3-1.7B-Wordle-SFT" batch_size = 512 group_size = 16 -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "wordle" env.taskset = { id = "wordle-v1" } env.player.harness = { id = "null" } diff --git a/k8s/prime-rl/examples/reverse-text/orch.toml b/k8s/prime-rl/examples/reverse-text/orch.toml index baa29bde5f..53049aa8c4 100644 --- a/k8s/prime-rl/examples/reverse-text/orch.toml +++ b/k8s/prime-rl/examples/reverse-text/orch.toml @@ -13,10 +13,11 @@ name = "reverse-text" [train.sampling] max_completion_tokens = 128 -[[train.env]] +[[train.source]] name = "reverse-text" -taskset = { id = "reverse-text-v1" } -harness = { id = "null", runtime = { type = "subprocess" } } +env.taskset = { id = "reverse-text-v1" } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } [renderer] name = "prime-qwen3" diff --git a/packages/prime-rl-configs/pyproject.toml b/packages/prime-rl-configs/pyproject.toml index 7f704768ab..4cd9728245 100644 --- a/packages/prime-rl-configs/pyproject.toml +++ b/packages/prime-rl-configs/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "renderers>=0.1.8", "tomli>=2.2.1", "tomli-w>=1.2.0", - "verifiers>=0.2.1", + "verifiers>=0.2.2.dev43", ] [build-system] diff --git a/packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py b/packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py index fe6b696203..f17720b9c1 100644 --- a/packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py +++ b/packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py @@ -3,7 +3,7 @@ from typing import Annotated, Any, Literal, TypeAlias import verifiers.v1 as vf -from pydantic import AliasChoices, Field, model_validator +from pydantic import AliasChoices, Field, SerializeAsAny, model_validator from renderers import AutoRendererConfig, RendererConfig from prime_rl.configs.algorithm import ( @@ -151,27 +151,52 @@ def _deprecate_max_tokens(cls, data: Any) -> Any: return data -class EnvConfig(vf.EnvServerConfig): - name: str | None = None - """Display name for this environment in logs, metrics, and buffer keys. Defaults to the taskset id. Must be unique across all envs in the same group.""" +class ServingConfig(vf.ServingConfig): + """Verifiers' serving block with ``address`` back to optional. Verifiers defaults it + to the address its own ``serve`` CLI binds; here the question is whether to spawn a + server or connect to one already running, and that answer has to survive the + resolved config being written to a file and read back — so it must be a *value* + (``None``), not field-set metadata, which a round-trip drops.""" address: str | None = None - """ZMQ address of an external env server (e.g. ``tcp://host:5000``). When set, the orchestrator connects to this server instead of spawning one; when None, a subprocess env server is spawned automatically. The ``pool`` sizes the spawned server.""" + """ZMQ address of an external env server (e.g. ``tcp://host:5000``). When set, the orchestrator connects to that server instead of spawning one; when None, it spawns a subprocess env server on a free port. ``pool`` sizes the spawned server.""" + + +class EnvConfig(BaseConfig): + """One environment a run pulls from: the verifiers blocks it composes (``env`` — what + runs, ``serve`` — how it's hosted, ``legacy`` — a classic v0 env instead) plus this + orchestrator's own per-env knobs.""" + + env: SerializeAsAny[vf.EnvConfig] = vf.SingleAgentEnvConfig() + """The verifiers environment — which env, its seed taskset, each agent, its knobs. Narrowed to the selected env's config class by the env id, else the taskset id.""" + + serve: ServingConfig = ServingConfig() + """How the env server is run: ``serve.pool`` sizes the spawned server, ``serve.address`` points at an external one instead, and ``serve.max_concurrent`` bounds one worker's episodes in flight (unset = unbounded; the dispatcher's ``max_inflight_episodes`` is the run's bound).""" + + legacy: vf.LegacyEnvConfig = vf.LegacyEnvConfig() + """A classic (v0) environment to run through the bridge instead of ``env``.""" + + name: str | None = None + """Display name for this environment in logs, metrics, and buffer keys. Defaults to the taskset id. Must be unique across all envs in the same group.""" ratio: float = Field(1.0, gt=0) """Sampling weight for this environment in the buffer. Relative weights are normalized to probabilities across envs (e.g. [1, 1] and [0.5, 0.5] are equivalent). Defaults to 1, i.e. equal weight per env.""" @model_validator(mode="before") @classmethod - def _migrate_num_workers(cls, data): - """Back-compat: the removed ``num_workers`` maps onto ``pool`` — an int becomes a - fixed ``static`` pool, ``"auto"`` falls through to the default ``elastic`` pool. An - explicit ``pool`` always wins.""" - if isinstance(data, dict) and "num_workers" in data: - num_workers = data.pop("num_workers") - if "pool" not in data and num_workers != "auto": - data["pool"] = {"type": "static", "num_workers": num_workers} - return data + def _resolve_env(cls, data): + """Narrow ``env`` to the selected env's config class.""" + return vf.resolve_env_field(data, vf.narrowed_env_annotation(cls)) + + @property + def is_legacy(self) -> bool: + """A classic (v0) env run through the bridge: a legacy id and no v1 taskset.""" + return self.legacy.id is not None and not self.env.taskset.id + + @property + def env_id(self) -> str: + """The env's identifier: the v1 env's, else the v0 env id.""" + return self.env.env_id or self.legacy.id or "" @property def resolved_name(self) -> str: @@ -179,9 +204,24 @@ def resolved_name(self) -> str: @model_validator(mode="after") def validate_env(self): + # A v0 id next to any v1 env identity leaves one of the two going nowhere, and + # which one depends on `is_legacy`: a taskset makes it False, so the v0 env never + # loads; a bare `env.id` leaves it True, so the v0 env runs under the v1 name. + if self.legacy.id is not None and self.env.env_id: + if self.env.taskset.id: + raise ValueError( + f"legacy.id {self.legacy.id!r} is a classic (v0) env and can't combine with " + f"the v1 taskset {self.env.taskset.id!r}. Pairing a reusable env with a taskset " + f"is env.id = {self.legacy.id!r}; to run the v0 env instead, drop the taskset." + ) + raise ValueError( + f"legacy.id {self.legacy.id!r} is a classic (v0) env and can't combine with the " + f"v1 env.id {self.env.id!r}: the v0 env is what would run, stamped with the v1 " + "env's name. Keep whichever one you meant to run." + ) if not self.env_id: raise ValueError( - 'no env configured — set env = { taskset = { id = "" } } (v1) or id = "" (v0/legacy)' + 'no env configured — set env = { taskset = { id = "" } } (v1) or legacy = { id = "" } (v0)' ) if self.resolved_name == "agg": raise ValueError( @@ -192,21 +232,21 @@ def validate_env(self): @model_validator(mode="after") def resolve_legacy_env_kwargs(self): """For a v0/legacy env, surface the v1 knobs the legacy bridge applies via - ``extra_env_kwargs`` (``env.set_kwargs(...)``): the per-rollout wall-clock timeout and - the multi-turn completion-token budget, read off ``env.agent``. (``max_seq_len`` is - added per train run in ``OrchestratorConfig.resolve_env_config``, which knows - ``seq_len``.)""" + ``legacy.extra_env_kwargs`` (``env.set_kwargs(...)``): the per-rollout wall-clock + timeout and the multi-turn completion-token budget, read off ``env.agent``. + (``max_seq_len`` is added per train run in ``OrchestratorConfig.resolve_env_config``, + which knows ``seq_len``.)""" if self.is_legacy: agent = getattr(self.env, "agent", None) if agent is not None: if agent.timeout.rollout is not None: - self.extra_env_kwargs["timeout_seconds"] = agent.timeout.rollout + self.legacy.extra_env_kwargs["timeout_seconds"] = agent.timeout.rollout if agent.max_output_tokens is not None: - self.extra_env_kwargs["max_total_completion_tokens"] = agent.max_output_tokens + self.legacy.extra_env_kwargs["max_total_completion_tokens"] = agent.max_output_tokens return self -class TrainEnvConfig(EnvConfig): +class TrainSourceConfig(EnvConfig): sampling: TrainSamplingConfig = TrainSamplingConfig() """Per-env sampling overrides. Unset fields inherit from the group-level train sampling config.""" @@ -220,7 +260,7 @@ class TrainEnvConfig(EnvConfig): this env its own algorithm.""" -class EvalEnvConfig(EnvConfig): +class EvalSourceConfig(EnvConfig): sampling: EvalSamplingConfig = EvalSamplingConfig() """Per-env sampling overrides. Unset fields inherit from the group-level eval sampling config.""" @@ -235,8 +275,8 @@ class EvalEnvConfig(EnvConfig): class TrainConfig(BaseConfig): - env: list[TrainEnvConfig] = Field(default_factory=list) - """Training environments.""" + source: list[TrainSourceConfig] = Field(default_factory=list) + """Training sources.""" sampling: TrainSamplingConfig = TrainSamplingConfig() """Shared training sampling configuration.""" @@ -246,7 +286,7 @@ def resolve_env_defaults(self): """Resolve per-env overrides: inherit group-level sampling (the worker ``pool`` is configured per env, defaulting to elastic).""" group_sampling = self.sampling.model_dump() - for env in self.env: + for env in self.source: if "sampling" not in env.model_fields_set: env.sampling = TrainSamplingConfig(**group_sampling) else: @@ -256,7 +296,7 @@ def resolve_env_defaults(self): @model_validator(mode="after") def validate_unique_env_names(self): - env_names = [env.resolved_name for env in self.env] + env_names = [env.resolved_name for env in self.source] duplicates = [n for n in env_names if env_names.count(n) > 1] if duplicates: raise ValueError( @@ -266,8 +306,8 @@ def validate_unique_env_names(self): class EvalConfig(BaseConfig): - env: list[EvalEnvConfig] = Field(default_factory=list) - """Evaluation environments.""" + source: list[EvalSourceConfig] = Field(default_factory=list) + """Evaluation sources.""" sampling: EvalSamplingConfig = Field(default_factory=EvalSamplingConfig) """Shared eval sampling configuration; can differ from training sampling.""" @@ -290,33 +330,33 @@ def resolve_env_defaults(self): """Resolve per-env overrides: inherit group-level sampling, num_examples, group_size, and interval (the worker ``pool`` is configured per env, default elastic).""" group_sampling = self.sampling.model_dump() - for env in self.env: - if "sampling" not in env.model_fields_set: - env.sampling = EvalSamplingConfig(**group_sampling) + for source in self.source: + if "sampling" not in source.model_fields_set: + source.sampling = EvalSamplingConfig(**group_sampling) else: - merged = group_sampling | env.sampling.model_dump(exclude_unset=True) - env.sampling = EvalSamplingConfig(**merged) - if "num_examples" not in env.model_fields_set: - env.num_examples = self.num_examples - if "group_size" not in env.model_fields_set: - env.group_size = self.group_size - if "interval" not in env.model_fields_set: - env.interval = self.interval + merged = group_sampling | source.sampling.model_dump(exclude_unset=True) + source.sampling = EvalSamplingConfig(**merged) + if "num_examples" not in source.model_fields_set: + source.num_examples = self.num_examples + if "group_size" not in source.model_fields_set: + source.group_size = self.group_size + if "interval" not in source.model_fields_set: + source.interval = self.interval return self @model_validator(mode="after") - def validate_non_empty_envs(self): - if not self.env: + def validate_non_empty_sources(self): + if not self.source: raise ValueError( - "EvalConfig must define at least one env. Either drop the " + "EvalConfig must define at least one source. Either drop the " "[orchestrator.eval] block entirely (to disable eval) or " - "add a [[orchestrator.eval.env]] block." + "add a [[orchestrator.eval.source]] block." ) return self @model_validator(mode="after") def validate_unique_env_names(self): - env_names = [env.resolved_name for env in self.env] + env_names = [source.resolved_name for source in self.source] duplicates = [n for n in env_names if env_names.count(n) > 1] if duplicates: raise ValueError( @@ -437,7 +477,7 @@ class OrchestratorConfig(BaseConfig): algo: AlgoConfig = GRPOAlgoConfig() """Training algorithm: sampling plus the per-token training signal (credit assignment and loss routing, fused — its ``type`` names the algorithm). - Defaults to ``grpo``. Override per env via ``[[orchestrator.train.env]]``'s + Defaults to ``grpo``. Override per source via ``[[orchestrator.train.source]]``'s ``algo``.""" model: ModelConfig = ModelConfig() @@ -525,10 +565,12 @@ class OrchestratorConfig(BaseConfig): """Tokens to train on per step (token-based batching). Set this OR ``batch_size``.""" oversampling_factor: float | None = Field(None, gt=0) - """Rollout-mode batching only. Multiplier used to derive ``max_inflight_rollouts`` from ``batch_size`` when ``max_inflight_rollouts`` is unset. Values below 1.0 intentionally cap in-flight rollout capacity below ``batch_size``.""" + """Rollout-mode batching only. Multiplier used to derive ``max_inflight_episodes`` from ``batch_size`` when ``max_inflight_episodes`` is unset. Values below 1.0 intentionally cap in-flight episode capacity below ``batch_size``.""" - max_inflight_rollouts: int | None = Field(None, ge=1) - """Maximum number of rollouts kept in-flight. Required for token-based batching. With ``batch_size`` set, defaults to ``batch_size * oversampling_factor`` (or ``batch_size`` when ``oversampling_factor`` is unset).""" + max_inflight_episodes: int | None = Field( + None, ge=1, validation_alias=AliasChoices("max_inflight_episodes", "max_inflight_rollouts") + ) + """Maximum number of episodes kept in-flight — one episode is one agent run at a time, whatever the env's agents are. Required for token-based batching. With ``batch_size`` set, defaults to ``batch_size * oversampling_factor`` (or ``batch_size`` when ``oversampling_factor`` is unset).""" group_size: int = Field(1, ge=1, validation_alias=AliasChoices("group_size", "rollouts_per_example")) """Output sequences returned per example during training.""" @@ -554,29 +596,20 @@ class OrchestratorConfig(BaseConfig): @model_validator(mode="before") @classmethod - def _env_to_train(cls, data: Any) -> Any: - """Allow [[env]] and [sampling] as shorthand for [train] with [[train.env]] and [train.sampling].""" + def _sampling_to_train(cls, data: Any) -> Any: + """Allow [sampling] as shorthand for [train.sampling].""" if not isinstance(data, dict): return data - if "env" in data or "sampling" in data: + if "sampling" in data: train = data.setdefault("train", {}) if isinstance(train, dict): - if "env" in data: - warnings.warn( - "'[[orchestrator.env]]' is deprecated, use '[[orchestrator.train.env]]' instead. " - "Auto-translating for now, but this will be removed in a future release.", - FutureWarning, - stacklevel=2, - ) - train.setdefault("env", data.pop("env")) - if "sampling" in data: - warnings.warn( - "'[orchestrator.sampling]' is deprecated, use '[orchestrator.train.sampling]' instead. " - "Auto-translating for now, but this will be removed in a future release.", - FutureWarning, - stacklevel=2, - ) - train.setdefault("sampling", data.pop("sampling")) + warnings.warn( + "'[orchestrator.sampling]' is deprecated, use '[orchestrator.train.sampling]' instead. " + "Auto-translating for now, but this will be removed in a future release.", + FutureWarning, + stacklevel=2, + ) + train.setdefault("sampling", data.pop("sampling")) return data @model_validator(mode="after") @@ -617,7 +650,7 @@ def validate_unique_filter_types(self): def inherit_env_algorithms(self): """Envs without their own algorithm inherit the top-level one. Declared before any validator that reads ``algo``.""" - for env_cfg in self.train.env: + for env_cfg in self.train.source: if env_cfg.algo is None: env_cfg.algo = self.algo.model_copy(deep=True) return self @@ -625,7 +658,7 @@ def inherit_env_algorithms(self): @property def any_policy_sourced(self) -> bool: """True when at least one train env samples rollouts from the live policy.""" - return any(env.algo is not None and env.algo.sampling.source == "policy" for env in self.train.env) + return any(env.algo is not None and env.algo.sampling.source == "policy" for env in self.train.source) @model_validator(mode="after") def validate_pool_size(self): @@ -691,29 +724,29 @@ def resolve_batching(self): if has_token_batch: if self.oversampling_factor is not None: raise ValueError("oversampling_factor can only be set when batch_size is set") - if self.max_inflight_rollouts is None: - raise ValueError("max_inflight_rollouts must be set when token_batch_size is set") + if self.max_inflight_episodes is None: + raise ValueError("max_inflight_episodes must be set when token_batch_size is set") else: assert self.batch_size is not None if self.batch_size % self.group_size != 0: raise ValueError("Batch size must be divisible by the number of samples per problem") oversampling_factor = self.oversampling_factor if self.oversampling_factor is not None else 1.0 - resolved_max_inflight_rollouts = max( + resolved_max_inflight_episodes = max( self.group_size, int(self.batch_size * oversampling_factor), ) - if self.max_inflight_rollouts is not None and self.oversampling_factor is not None: - expected_max_inflight_rollouts = resolved_max_inflight_rollouts - if self.max_inflight_rollouts != expected_max_inflight_rollouts: - raise ValueError("max_inflight_rollouts conflicts with oversampling_factor * batch_size") - if self.max_inflight_rollouts is None: - self.max_inflight_rollouts = resolved_max_inflight_rollouts + if self.max_inflight_episodes is not None and self.oversampling_factor is not None: + expected_max_inflight_episodes = resolved_max_inflight_episodes + if self.max_inflight_episodes != expected_max_inflight_episodes: + raise ValueError("max_inflight_episodes conflicts with oversampling_factor * batch_size") + if self.max_inflight_episodes is None: + self.max_inflight_episodes = resolved_max_inflight_episodes - if self.max_inflight_rollouts is not None and self.max_inflight_rollouts < self.group_size: - raise ValueError("max_inflight_rollouts must be at least the number of rollouts per example") + if self.max_inflight_episodes is not None and self.max_inflight_episodes < self.group_size: + raise ValueError("max_inflight_episodes must be at least the number of rollouts per example") # Propagate the top-level ``group_size`` into each train env that didn't set its own. - for env_cfg in self.train.env: + for env_cfg in self.train.source: if "group_size" not in env_cfg.model_fields_set: env_cfg.group_size = self.group_size @@ -736,7 +769,7 @@ def auto_setup_bench(self): @model_validator(mode="after") def resolve_env_config(self): """Set vLLM sampling defaults + legacy env kwargs on each train env from top-level fields.""" - for env in self.train.env: + for env in self.train.source: # Policy-sourced rollouts hit our vLLM server; frozen-sourced # rollouts may hit external OAI endpoints that reject these knobs. assert env.algo is not None @@ -746,6 +779,6 @@ def resolve_env_config(self): env.sampling.extra_body.setdefault("return_token_ids", True) if env.is_legacy: # v0 env: cap per-turn response tokens to the training budget (the legacy - # bridge applies extra_env_kwargs via env.set_kwargs). - env.extra_env_kwargs["max_seq_len"] = self.seq_len + # bridge applies legacy.extra_env_kwargs via env.set_kwargs). + env.legacy.extra_env_kwargs["max_seq_len"] = self.seq_len return self diff --git a/pyproject.toml b/pyproject.toml index d6033582b8..18c7f1baca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "setproctitle>=1.3.0", "uvloop>=0.21.0", "torchtitan", - "verifiers[harbor]>=0.2.1", + "verifiers[harbor]>=0.2.2.dev43", "renderers", "dion", "tilelang>=0.1.8", diff --git a/skills/configs/SKILL.md b/skills/configs/SKILL.md index 887340cfb3..9672da52cb 100644 --- a/skills/configs/SKILL.md +++ b/skills/configs/SKILL.md @@ -41,20 +41,26 @@ Incompatible combinations (e.g. CP requires flash attention) must raise in a `mo **Lists** — TOML uses array of tables; later config files replace lists wholesale, so overlays must include the full desired list: ```toml -[[orchestrator.train.env]] +[[orchestrator.train.source]] name = "reverse-text" env.taskset = { id = "reverse-text-v1" } env.agent.harness = { id = "null" } env.agent.runtime = { type = "subprocess" } + +[[orchestrator.eval.source]] +name = "reverse-text-eval" +env.taskset = { id = "reverse-text-v1", split = "test" } +env.agent.harness = { id = "null" } +env.agent.runtime = { type = "subprocess" } ``` -CLI: `--orchestrator.train.env.0.env.taskset.id reverse-text-v1`. +CLI: `--orchestrator.train.source.0.env.taskset.id reverse-text-v1` or `--orchestrator.eval.source.0.env.taskset.id reverse-text-v1`. **Dicts** — TOML uses a section; CLI takes a JSON string: `--vllm-extra '{"key1": "value1"}'`. This works for plain `dict` fields only — nested pydantic-model fields (e.g. `algo`) reject JSON strings; use dotted keys (`--orchestrator.algo.type max_rl`) or a TOML overlay file. **Discriminated unions** — set the `type` field to pick the variant (`[orchestrator.algo] type = "max_rl"`). Omit `type` to keep the default variant. -**Algorithms** — `[orchestrator.algo] type = "grpo" | "max_rl" | "rae" | "opd" | "opsd" | "sft" | "echo"` — the type names the algorithm (credit assignment + loss routing, fused), and each type's class defaults are its vetted setting; any other key you set is your own assembly (e.g. `[orchestrator.algo.roles.user] alpha = 0.1` for echo — setting any echo role replaces the whole role table). There is no preset layer, and no config hook that points at user code — a new algorithm is a named class in the repo (subclass `Algorithm`, register it). Per-env override: `[orchestrator.train.env.algo] type = "opd"` (the env assembles its own algorithm). prime-rl only hosts the trainable policy; frozen models are inline external endpoints on the algorithm, named where the model is used — `[orchestrator.algo.teacher]` for opd (the frozen model scored against), `[orchestrator.algo.sampling.source]` for sft (the model it samples from), each with `name` + `base_url`. There is no shared `teacher` slot. opsd declares no model — it self-distills against the live policy. See `docs/algorithms.md`. +**Algorithms** — `[orchestrator.algo] type = "grpo" | "max_rl" | "rae" | "opd" | "opsd" | "sft" | "echo"` — the type names the algorithm (credit assignment + loss routing, fused), and each type's class defaults are its vetted setting; any other key you set is your own assembly (e.g. `[orchestrator.algo.roles.user] alpha = 0.1` for echo — setting any echo role replaces the whole role table). There is no preset layer, and no config hook that points at user code — a new algorithm is a named class in the repo (subclass `Algorithm`, register it). Per-source override: `[orchestrator.train.source.algo] type = "opd"` (the source assembles its own algorithm). prime-rl only hosts the trainable policy; frozen models are inline external endpoints on the algorithm, named where the model is used — `[orchestrator.algo.teacher]` for opd (the frozen model scored against), `[orchestrator.algo.sampling.source]` for sft (the model it samples from), each with `name` + `base_url`. There is no shared `teacher` slot. opsd declares no model — it self-distills against the live policy. See `docs/algorithms.md`. **`BaseModel | None` fields** — bare flag enables defaults; nested override enables and sets: diff --git a/src/prime_rl/entrypoints/rl.py b/src/prime_rl/entrypoints/rl.py index 08e9f481ed..c58eafc915 100644 --- a/src/prime_rl/entrypoints/rl.py +++ b/src/prime_rl/entrypoints/rl.py @@ -201,9 +201,9 @@ def sigterm_handler(signum, frame): ) frozen_endpoints: list[str] = [] - for env in config.orchestrator.train.env: + for env in config.orchestrator.train.source: algo = env.algo - assert algo is not None, "TrainEnvConfig.algo must be resolved before launch (inherit_env_algorithms)" + assert algo is not None, "TrainSourceConfig.algo must be resolved before launch (inherit_env_algorithms)" for ref in (algo.sampling.source, getattr(algo, "teacher", None)): if isinstance(ref, FrozenModelConfig): frozen_endpoints.append(f"{ref.name} ({', '.join(ref.base_url)})") @@ -479,8 +479,10 @@ def rl_slurm(config: RLConfig): write_config(config, config_dir, exclude={"slurm", "dry_run", "clean_output_dir"}) logger.info(f"Wrote config to {config_dir / RL_TOML}") - train_env_names = [env.resolved_name for env in config.orchestrator.train.env] - eval_env_names = [env.resolved_name for env in config.orchestrator.eval.env] if config.orchestrator.eval else [] + train_env_names = [env.resolved_name for env in config.orchestrator.train.source] + eval_env_names = ( + [source.resolved_name for source in config.orchestrator.eval.source] if config.orchestrator.eval else [] + ) log_message = format_log_message( log_dir=log_dir, @@ -494,8 +496,10 @@ def rl_slurm(config: RLConfig): write_subconfigs(config, config_dir) logger.info(f"Wrote subconfigs to {config_dir}") - train_env_names = [env.resolved_name for env in config.orchestrator.train.env] - eval_env_names = [env.resolved_name for env in config.orchestrator.eval.env] if config.orchestrator.eval else [] + train_env_names = [env.resolved_name for env in config.orchestrator.train.source] + eval_env_names = ( + [source.resolved_name for source in config.orchestrator.eval.source] if config.orchestrator.eval else [] + ) has_infer = config.deployment.infer_nodes_per_replica > 0 log_message = format_log_message( diff --git a/src/prime_rl/orchestrator/dispatcher.py b/src/prime_rl/orchestrator/dispatcher.py index 998ff6fd53..10def1dbde 100644 --- a/src/prime_rl/orchestrator/dispatcher.py +++ b/src/prime_rl/orchestrator/dispatcher.py @@ -1,7 +1,9 @@ """RolloutDispatcher: schedules rollouts under a shared permit counter. -- Capacity (``max_inflight_rollouts``) is shared across train + eval. - A group-scoring task that runs N rollouts in one call reserves N permits. +- Capacity (``max_inflight_episodes``) is shared across train + eval. One permit is + one episode: for a v1 env one ``run`` request, and a group-scoring task that runs + N rollouts in one call reserves N permits (each bridged v0 rollout is its own + single-agent episode). - Optional rate limiting via ``AsyncLimiter(tasks_per_minute, 60)``. - Emit-everything invariant: every dispatched env-rollout eventually reaches ``out_q`` exactly once, as one episode (a ``list[Rollout]``). Failures @@ -129,7 +131,7 @@ def __init__( eval_source: EvalSource | None, policy_pool: InferencePool, policy: Policy, - max_inflight_rollouts: int, + max_inflight_episodes: int, tasks_per_minute: float | None, max_off_policy_steps: int, ) -> None: @@ -143,7 +145,7 @@ def __init__( self.eval_source = eval_source self.max_off_policy_steps = max_off_policy_steps - self.max_inflight = max_inflight_rollouts + self.max_inflight = max_inflight_episodes self.inflight_permits = 0 self.rate_limiter: AsyncLimiter | None = ( AsyncLimiter(tasks_per_minute, time_period=60) if tasks_per_minute else None diff --git a/src/prime_rl/orchestrator/env_server/env_server.py b/src/prime_rl/orchestrator/env_server/env_server.py index 19ad321bc4..5b911bfac2 100644 --- a/src/prime_rl/orchestrator/env_server/env_server.py +++ b/src/prime_rl/orchestrator/env_server/env_server.py @@ -13,18 +13,18 @@ @clean_exit def run_server(config: EnvServerConfig): env = config.env - address = env.address or "tcp://127.0.0.1:5000" - # The env's ``pool`` (static or elastic) sizes the server; a v0/legacy env runs through + address = env.serve.address or "tcp://127.0.0.1:5000" + # The env's ``serve.pool`` (static or elastic) sizes the server; a v0/legacy env runs through # the bridge, a v1 env is a native env block — both speak the same serve protocol, # so the orchestrator is agnostic. serve_env applies the logging setup in this process # and in every spawned worker. server_kwargs = ( - {"env_id": env.env_id, "env_args": env.args, "extra_env_kwargs": env.extra_env_kwargs} + {"env_id": env.env_id, "env_args": env.legacy.args, "extra_env_kwargs": env.legacy.extra_env_kwargs} if env.is_legacy - else {"config_data": env_config_data(env.env)} + else {"config_data": env_config_data(env.env), "max_concurrent": env.serve.max_concurrent} ) serve_env( - **pool_serve_kwargs(env.pool), + **pool_serve_kwargs(env.serve.pool), legacy=env.is_legacy, address=address, log_setup=partial(setup_env_server_logging, config.log.level, config.log.json_logging), diff --git a/src/prime_rl/orchestrator/envs.py b/src/prime_rl/orchestrator/envs.py index 2cae9d185f..3bffb3504d 100644 --- a/src/prime_rl/orchestrator/envs.py +++ b/src/prime_rl/orchestrator/envs.py @@ -1,7 +1,7 @@ """Env wrappers over a v1 env server. Each ``Env`` owns a v1 ``EnvServer`` (spawned as a child process, or an -external one given by ``config.address``) and an ``EnvClient`` to drive it. The +external one pinned by ``config.serve.address``) and an ``EnvClient`` to drive it. The orchestrator never *runs* an environment — the agents and their runtimes live only in the server — but it does own the *taskset*: a v1 env's tasks are loaded here, once, and each dispatched env-rollout ships its task's data on the request @@ -34,7 +34,7 @@ import verifiers.v1 as vf from verifiers.v1.serve import EnvClient, env_config_data -from prime_rl.configs.orchestrator import EnvConfig, EvalEnvConfig, TrainEnvConfig +from prime_rl.configs.orchestrator import EnvConfig, EvalSourceConfig, TrainSourceConfig from prime_rl.orchestrator.algo import Algorithm, build_algorithm from prime_rl.orchestrator.sampler import Sampler from prime_rl.orchestrator.types import Rollout @@ -112,8 +112,8 @@ def env_client(self) -> EnvClient: async def start(self, log_dir: Path, log_level: str | None = None, json_logging: bool = False) -> None: """Spawn the env server (if needed), connect, and load the taskset client-side (legacy instead asks the server for ``info`` — its dataset is server-side).""" - external = self.config.address is not None - address = self.config.address or await self._spawn(log_dir, log_level or "INFO", json_logging) + external = self.config.serve.address is not None + address = self.config.serve.address or await self._spawn(log_dir, log_level or "INFO", json_logging) get_logger().debug(f"Connecting {self.name} to env server {address}") self._env_client = EnvClient(address=address) # A spawned server already reported its address *after* binding, so it's up. An @@ -152,12 +152,18 @@ async def _spawn(self, log_dir: Path, log_level: str, json_logging: bool) -> str dict( legacy=True, env_id=self.config.env_id, - env_args=self.config.args, - extra_env_kwargs=self.config.extra_env_kwargs, + env_args=self.config.legacy.args, + extra_env_kwargs=self.config.legacy.extra_env_kwargs, ) if self.config.is_legacy # Picklable dict — the narrowed config class doesn't survive the spawn. - else dict(legacy=False, config_data=env_config_data(self.config.env)) + # ``max_concurrent`` bounds this worker's episodes in flight — usually unset, + # since the dispatcher's ``max_inflight_episodes`` is the run's bound. + else dict( + legacy=False, + config_data=env_config_data(self.config.env), + max_concurrent=self.config.serve.max_concurrent, + ) ) process = ctx.Process( target=_run_env_server, @@ -165,7 +171,7 @@ async def _spawn(self, log_dir: Path, log_level: str, json_logging: bool) -> str log_file=str(log_file), log_level=log_level, json_logging=json_logging, - **vf.pool_serve_kwargs(self.config.pool), + **vf.pool_serve_kwargs(self.config.serve.pool), address="tcp://127.0.0.1:0", address_queue=address_queue, **server_kwargs, @@ -246,9 +252,9 @@ def shutdown(self) -> None: class TrainEnv(Env): - config: TrainEnvConfig + config: TrainSourceConfig - def __init__(self, config: TrainEnvConfig, sampler: Sampler, algorithm: Algorithm): + def __init__(self, config: TrainSourceConfig, sampler: Sampler, algorithm: Algorithm): super().__init__(config) self.sampler = sampler self.algorithm = algorithm @@ -256,9 +262,9 @@ def __init__(self, config: TrainEnvConfig, sampler: Sampler, algorithm: Algorith class EvalEnv(Env): - config: EvalEnvConfig + config: EvalSourceConfig - def __init__(self, config: EvalEnvConfig): + def __init__(self, config: EvalSourceConfig): super().__init__(config) self.sampling_args = config.sampling.to_sampling_args() self.examples: list[dict] = [] @@ -333,10 +339,10 @@ class TrainEnvs(Envs[TrainEnv]): :class:`Sampler` and runtime :class:`Algorithm`, built from the env's resolved algorithm config.""" - def __init__(self, configs: Sequence[TrainEnvConfig], *, policy_pool, renderer_config=None): + def __init__(self, configs: Sequence[TrainSourceConfig], *, policy_pool, renderer_config=None): self._envs: dict[str, TrainEnv] = {} for config in configs: - assert config.algo is not None, "TrainEnvConfig.algo must be resolved before env construction" + assert config.algo is not None, "TrainSourceConfig.algo must be resolved before env construction" env = TrainEnv( config, Sampler(config.algo.sampling, policy_pool, renderer_config), @@ -348,7 +354,7 @@ def __init__(self, configs: Sequence[TrainEnvConfig], *, policy_pool, renderer_c class EvalEnvs(Envs[EvalEnv]): """Collection of evaluation environments.""" - def __init__(self, configs: Sequence[EvalEnvConfig]): + def __init__(self, configs: Sequence[EvalSourceConfig]): self._envs: dict[str, EvalEnv] = {} for config in configs: env = EvalEnv(config) diff --git a/src/prime_rl/orchestrator/orchestrator.py b/src/prime_rl/orchestrator/orchestrator.py index 400d078f79..ae063c383a 100644 --- a/src/prime_rl/orchestrator/orchestrator.py +++ b/src/prime_rl/orchestrator/orchestrator.py @@ -156,7 +156,7 @@ def __init__(self, config: OrchestratorConfig) -> None: # Route the in-process v1 library logging through our handler. The # env server runs in a child process, so its logging is separate. intercept_vf_logging(logger="verifiers.v1", level="WARN") - algorithms = sorted({env.algo.type for env in config.train.env if env.algo is not None}) + algorithms = sorted({env.algo.type for env in config.train.source if env.algo is not None}) get_logger().info(f"Starting orchestrator (algorithm: {', '.join(algorithms)})") if config.bench: @@ -238,8 +238,8 @@ async def setup(self) -> None: tokenizer=self.tokenizer, run_config=config, keep_full_history=config.bench, - train_env_names=[env.resolved_name for env in config.train.env], - eval_env_names=[env.resolved_name for env in config.eval.env] if config.eval is not None else [], + train_env_names=[env.resolved_name for env in config.train.source], + eval_env_names=[source.resolved_name for source in config.eval.source] if config.eval is not None else [], ) if config.heartbeat is not None: @@ -256,7 +256,7 @@ async def setup(self) -> None: get_logger().info("Loading training environments") self.train_envs = TrainEnvs( - config.train.env, policy_pool=self.policy_inference, renderer_config=config.renderer + config.train.source, policy_pool=self.policy_inference, renderer_config=config.renderer ) get_logger().debug( f"Loaded {len(self.train_envs)} training environment(s) ({', '.join(self.train_envs.names)})" @@ -270,7 +270,7 @@ async def setup(self) -> None: if config.eval is not None: get_logger().info("Loading eval environment(s)") - self.eval_envs = EvalEnvs(config.eval.env) + self.eval_envs = EvalEnvs(config.eval.source) get_logger().debug(f"Loaded {len(self.eval_envs)} eval environment(s) ({', '.join(self.eval_envs.names)})") await self.eval_envs.start( log_dir=get_log_dir(config.output_dir.parent) / "envs" / "eval", @@ -391,7 +391,7 @@ async def setup(self) -> None: else None ) - assert config.max_inflight_rollouts is not None, "max_inflight_rollouts must be resolved before dispatcher init" + assert config.max_inflight_episodes is not None, "max_inflight_episodes must be resolved before dispatcher init" log_interval = config.log.interval wandb_enabled = config.wandb is not None self.dispatcher = RolloutDispatcher( @@ -401,7 +401,7 @@ async def setup(self) -> None: eval_source=self.eval_source, policy_pool=self.policy_inference, policy=self.policy, - max_inflight_rollouts=config.max_inflight_rollouts, + max_inflight_episodes=config.max_inflight_episodes, tasks_per_minute=config.tasks_per_minute, max_off_policy_steps=config.max_off_policy_steps, ) diff --git a/src/prime_rl/utils/monitor/prime.py b/src/prime_rl/utils/monitor/prime.py index 374576ff91..e0d0afec5b 100644 --- a/src/prime_rl/utils/monitor/prime.py +++ b/src/prime_rl/utils/monitor/prime.py @@ -168,7 +168,7 @@ def _register_run(self, config: PrimeMonitorConfig, run_config: OrchestratorConf payload["batch_size"] = run_config.batch_size payload["rollouts_per_example"] = run_config.group_size payload["seq_len"] = run_config.seq_len - payload["environments"] = [{"id": env.id} for env in run_config.train.env] + payload["environments"] = [{"id": env.env_id} for env in run_config.train.source] payload["run_config"] = run_config.model_dump(exclude_none=True, mode="json") if run_config.wandb: payload["wandb_project"] = run_config.wandb.project diff --git a/tests/unit/test_configs.py b/tests/unit/test_configs.py index df50c6db4e..0d2e22f40d 100644 --- a/tests/unit/test_configs.py +++ b/tests/unit/test_configs.py @@ -25,11 +25,14 @@ def get_config_files() -> list[Path]: - """Any TOML file inside `configs/` or `examples/`.""" + """Any TOML file inside `configs/`, `examples/` or `k8s/`.""" config_files = list(Path("configs").rglob("*.toml")) example_files = list(Path("examples").rglob("*.toml")) + # The k8s example configs are mounted into the chart's containers verbatim, so a + # stale key there breaks a deploy with nothing else to catch it. + k8s_files = list(Path("k8s").rglob("*.toml")) - return config_files + example_files + return config_files + example_files + k8s_files @pytest.mark.parametrize("config_file", get_config_files(), ids=lambda x: x.as_posix()) @@ -177,10 +180,10 @@ def test_env_algo_overrides_top_level(): { "renderer": {"name": "qwen3"}, # echo needs the renderer's role attribution "algo": {"type": "echo"}, - "train": {"env": [{"id": "a", "algo": {"type": "grpo"}}, {"id": "b"}]}, + "train": {"source": [{"legacy": {"id": "a"}, "algo": {"type": "grpo"}}, {"legacy": {"id": "b"}}]}, } ) - env_a, env_b = config.train.env + env_a, env_b = config.train.source # Env a sets its own algorithm; only env b inherits the top-level echo algorithm. assert env_a.algo is not None and env_a.algo.type == "grpo" assert env_b.algo is not None and env_b.algo.type == "echo" @@ -188,7 +191,23 @@ def test_env_algo_overrides_top_level(): # Resolved configs round-trip. dumped = config.model_dump(exclude_none=True) reloaded = OrchestratorConfig.model_validate(dumped) - assert reloaded.train.env[0].algo is not None and reloaded.train.env[0].algo.type == "grpo" + assert reloaded.train.source[0].algo is not None and reloaded.train.source[0].algo.type == "grpo" + + with pytest.raises(ValidationError, match="env"): + OrchestratorConfig.model_validate( + { + "renderer": {"name": "qwen3"}, + "train": {"env": [{"legacy": {"id": "removed"}}]}, + } + ) + + with pytest.raises(ValidationError, match="env"): + OrchestratorConfig.model_validate( + { + "renderer": {"name": "qwen3"}, + "eval": {"env": [{"legacy": {"id": "removed"}}]}, + } + ) def test_trainer_enable_token_export_cli_flag(): diff --git a/tests/unit/train/rl/test_packer.py b/tests/unit/train/rl/test_packer.py index 890e290d20..81af6ae78c 100644 --- a/tests/unit/train/rl/test_packer.py +++ b/tests/unit/train/rl/test_packer.py @@ -32,7 +32,7 @@ def create_run_with_config(output_dir: Path, run_name: str) -> Path: "model": {"name": "test-model"}, "batch_size": 2, "group_size": 1, - "env": [{"id": "test-env"}], + "train": {"source": [{"legacy": {"id": "test-env"}}]}, "sampling": {"temperature": 1.0}, # test-model isn't in MODEL_RENDERER_MAP; use the explicit default renderer. "renderer": {"name": "default"}, diff --git a/tests/unit/train/test_runs.py b/tests/unit/train/test_runs.py index f650541be9..5fa819fc9c 100644 --- a/tests/unit/train/test_runs.py +++ b/tests/unit/train/test_runs.py @@ -41,7 +41,7 @@ def create_run_with_config( "model": {"name": "test-model"}, "batch_size": 32, "group_size": 4, - "env": [{"id": "test-env"}], + "train": {"source": [{"legacy": {"id": "test-env"}}]}, # test-model isn't in MODEL_RENDERER_MAP; use the explicit default renderer. "renderer": {"name": "default"}, } @@ -202,7 +202,7 @@ def test_config_loading(tmp_path: Path) -> None: "batch_size": 32, "max_steps": 1000, "group_size": 4, - "env": [{"id": "test-env"}], + "train": {"source": [{"legacy": {"id": "test-env"}}]}, "renderer": {"name": "default"}, } create_run_with_config(tmp_path, "run_test123", config=test_config) @@ -247,7 +247,7 @@ def test_config_cleanup_on_deletion(tmp_path: Path) -> None: "model": {"name": "test-model"}, "batch_size": 16, "group_size": 4, - "env": [{"id": "test-env"}], + "train": {"source": [{"legacy": {"id": "test-env"}}]}, "renderer": {"name": "default"}, } run_dir = create_run_with_config(tmp_path, "run_delete_me", config=test_config) @@ -278,7 +278,7 @@ def test_config_invalid(tmp_path: Path) -> None: "model": {"name": "test-model"}, "batch_size": "not-a-number", # Invalid type "group_size": 4, - "env": [{"id": "test-env"}], + "train": {"source": [{"legacy": {"id": "test-env"}}]}, } run_dir = create_run_with_config(tmp_path, "run_invalid", config=invalid_config) config_dir = run_dir / "control" diff --git a/uv.lock b/uv.lock index c6002b00a2..6145aa0369 100644 --- a/uv.lock +++ b/uv.lock @@ -936,21 +936,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e6/ec/0c42039e80b9acc534f67b73b7a42471948042859b3a64867b50a4a77fa3/chromadb-1.5.9-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc09b3df76e5a5cb386aed2715a2eea152e3949f9e1ba93c7119505377749929", size = 23316203, upload-time = "2026-05-05T05:54:41.157Z" }, ] -[[package]] -name = "claude-agent-sdk" -version = "0.2.107" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "mcp" }, - { name = "sniffio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/f1/34820c3b12ec3d10d1cf4b72a877958134e4e2e7bd5af4dbc0cf4f55df33/claude_agent_sdk-0.2.107.tar.gz", hash = "sha256:e1310eef43b886c66ac157b9e80af0fb6ccedd83ad6acc9d4a6bb59cffdd2620", size = 255641, upload-time = "2026-06-22T20:51:00.429Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/f1/ae6db2aa82f4d10ffa2c3c6953c2079d969f43e45bbf5f1ba3e5e6c66dad/claude_agent_sdk-0.2.107-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:abda89a41a4bcf2a9e1a3a94edf165ea5632174878874fcbfeeadbcc40e7f7d2", size = 73397534, upload-time = "2026-06-22T20:51:10.696Z" }, - { url = "https://files.pythonhosted.org/packages/9e/91/24808205b576842fb1ea8856d92da6ddca2903e1c3582e16d0ff0ed1b9d9/claude_agent_sdk-0.2.107-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:e3eabda428cbb1da71f1f0d3c742f81b1fe36c44d84e744fde39c4b9dbd3781b", size = 74371415, upload-time = "2026-06-22T20:51:14.154Z" }, -] - [[package]] name = "clbench-v1" version = "0.2.0" @@ -1893,11 +1878,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.29.0" +version = "3.32.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/80/8232b582c4b318b817cf1274ba74976b07b34d35ef439b3eb948f98645a1/filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402", size = 213757, upload-time = "2026-07-21T13:17:42.898Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, + { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, ] [[package]] @@ -2411,19 +2396,20 @@ wheels = [ [[package]] name = "harbor" -version = "0.14.0" +version = "0.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "claude-agent-sdk" }, - { name = "datasets" }, { name = "dirhash" }, { name = "fastapi" }, + { name = "filelock" }, { name = "httpx" }, { name = "jinja2" }, { name = "litellm" }, { name = "packaging" }, { name = "pathspec" }, + { name = "platformdirs" }, { name = "pydantic" }, + { name = "pyjwt" }, { name = "python-dotenv" }, { name = "pyyaml" }, { name = "requests" }, @@ -2435,9 +2421,9 @@ dependencies = [ { name = "typer" }, { name = "uvicorn" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a9/f3/cba38a11b0cca01ecd471437869f1cd57d86a3b1e4091b8fd3d112cba8af/harbor-0.14.0.tar.gz", hash = "sha256:0ffffc25a618e4b7bf2b901a8c717c8bc28877f5aa8c37be47f1ec8c301cb0b6", size = 1241271, upload-time = "2026-06-17T16:43:23.495Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/da/5a26998c6e7d9455321ab39bc8e9122993892ece13c3ad4f2b0de986aaf6/harbor-0.20.0.tar.gz", hash = "sha256:e2e5e88f772690fd121553ca34fd5d6dd6b4aaa51c8fae635abc84b112303112", size = 1590870, upload-time = "2026-07-18T21:25:22.578Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/22/cb447121af474cfc7662b9f997be20316eadc692f19fa884608460589020/harbor-0.14.0-py3-none-any.whl", hash = "sha256:171971a008386ce633ef9f53fb4170e7d7807dba15349e930366581dd4c511e2", size = 1412739, upload-time = "2026-06-17T16:43:25.062Z" }, + { url = "https://files.pythonhosted.org/packages/76/03/b6617f32385295729f3af0ae0d512cf87ba4793b9ce462ea020d776a9025/harbor-0.20.0-py3-none-any.whl", hash = "sha256:4b7e48223aea2384cdb8c9eff35eaebd482fc9b1ec09f8193a121c47356ff19a", size = 1792416, upload-time = "2026-07-18T21:25:19.206Z" }, ] [[package]] @@ -5020,11 +5006,11 @@ wheels = [ [[package]] name = "platformdirs" -version = "4.9.6" +version = "4.11.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" } +sdist = { url = "https://files.pythonhosted.org/packages/78/9b/560e4be8e26f6fd133a03630a8df0c663b9e8d61b4ade152b72005aec83b/platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0", size = 31953, upload-time = "2026-07-21T13:09:36.565Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, + { url = "https://files.pythonhosted.org/packages/7d/68/d8d58938dfb1370b266a1a729e6d77a985be23689a0496498ee17b2cbf90/platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74", size = 23247, upload-time = "2026-07-21T13:09:35.422Z" }, ] [[package]] @@ -5368,7 +5354,7 @@ requires-dist = [ { name = "renderers", specifier = ">=0.1.8" }, { name = "tomli", specifier = ">=2.2.1" }, { name = "tomli-w", specifier = ">=1.2.0" }, - { name = "verifiers", specifier = ">=0.2.1" }, + { name = "verifiers", specifier = ">=0.2.2.dev43" }, ] [[package]] @@ -7893,7 +7879,7 @@ requires-dist = [ { name = "anthropic", specifier = ">=0.78.0" }, { name = "datasets", specifier = ">=3.3.0,<6.0.0" }, { name = "gepa", specifier = ">=0.0.6" }, - { name = "harbor", marker = "python_full_version >= '3.12' and extra == 'harbor'", specifier = "==0.14.0" }, + { name = "harbor", marker = "python_full_version >= '3.12' and extra == 'harbor'", specifier = "==0.20.0" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "loguru", specifier = ">=0.7.0" }, { name = "math-verify", specifier = ">=0.8.0" },