Unify training inference through Chat Completions

configs/advanced/minimax-m2.5/swe.toml

@@ -64,5 +64,5 @@ env.agent.runtime.labels = ["minimax-swe", "swe-bench-verified"]
 
 [inference.vllm]
 tensor_parallel_size = 8
-[orchestrator.renderer]
+[inference.renderer]
 name = "minimax-m2"

configs/advanced/nemotron-3-super/swe.toml

@@ -72,7 +72,7 @@ num_output_tokens_weight = 0.0
 num_input_tokens_weight = 0.1
 num_turns_weight = 0.1
 
-[orchestrator.renderer]
+[inference.renderer]
 name = "nemotron-3"
 truncate_history_thinking = false
 

configs/ci/integration/alphabet_sort.toml

@@ -43,6 +43,6 @@ env.agent.runtime.type = "subprocess"
 
 # Mirror of Qwen/Qwen3-0.6B; PI's template patch always re-emits prior
 # <think> blocks. Match that with renderer thinking_retention.
-[orchestrator.renderer]
+[inference.renderer]
 name = "auto"
 thinking_retention = "all"

configs/ci/integration/reverse-text-lora/resume.toml

@@ -41,5 +41,5 @@ env.agent.runtime.type = "subprocess"
 [inference.vllm]
 enable_lora = true
 gpu_memory_utilization = 0.7
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"

configs/ci/integration/reverse-text-lora/start.toml

@@ -38,5 +38,5 @@ env.agent.runtime.type = "subprocess"
 [inference.vllm]
 enable_lora = true
 gpu_memory_utilization = 0.7
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"

configs/ci/integration/reverse-text-moe/start.toml

@@ -38,5 +38,5 @@ env.agent.runtime.type = "subprocess"
 [inference.vllm]
 max_model_len = 2048
 gpu_memory_utilization = 0.7
-[orchestrator.renderer]
+[inference.renderer]
 name = "default"

configs/ci/integration/reverse-text-rl-opd/start.toml

@@ -36,7 +36,7 @@ type = "opd"
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
 base_url = "http://localhost:8001/v1"
 
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"
 
 [orchestrator.train.sampling]

configs/ci/integration/reverse-text-rl-sft/start.toml

@@ -37,12 +37,8 @@ type = "sft"
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
 base_url = "http://localhost:8001/v1"
 
-# The student's renderer tokenizes the teacher's messages into the student's
-# token space (backfill). Use `prime-qwen3` — it matches this PrimeIntellect
-# Qwen3 model's chat template token-for-token (single-turn + system), so the CE
-# target stays faithful, without the empty `<think></think>` block the stock
-# `qwen3` renderer injects (which this model's template never emits).
-[orchestrator.renderer]
+# Match the policy chat template. The frozen generation server also owns its renderer.
+[inference.renderer]
 name = "prime-qwen3"
 
 [orchestrator.train.sampling]

configs/ci/integration/reverse-text/resume.toml

@@ -40,5 +40,5 @@ env.agent.runtime.type = "subprocess"
 
 # PrimeIntellect Qwen3 fine-tune - select its renderer explicitly (the -SFT
 # suffix misses the MODEL_RENDERER_MAP exact match).
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"

configs/ci/integration/reverse-text/start.toml

@@ -37,5 +37,5 @@ env.agent.runtime.type = "subprocess"
 
 # PrimeIntellect Qwen3 fine-tune - select its renderer explicitly (the -SFT
 # suffix misses the MODEL_RENDERER_MAP exact match).
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"

configs/ci/nightly-fft/hendrycks-sanity.toml

@@ -50,6 +50,7 @@ attn = "flash_attention_2"
 [inference.vllm]
 max_model_len = 8192
 data_parallel_size = 4
-[orchestrator.renderer]
+reasoning_parser = "qwen3"
+
+[inference.renderer]
 name = "default"
-reasoning_parser = "think"

configs/ci/nightly-fft/reverse-text.toml

@@ -36,5 +36,5 @@ max_completion_tokens = 128
 
 [inference]
 
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"

configs/ci/nightly-fft/wordle.toml

@@ -36,5 +36,5 @@ max_completion_tokens = 1024
 
 [inference.vllm]
 data_parallel_size = 6
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"

configs/debug/algo/echo.toml

@@ -39,7 +39,7 @@ env.agent.runtime.type = "subprocess"
 max_completion_tokens = 512
 
 # Fine-tune inherits the PrimeIntellect Qwen3 template byte-for-byte.
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"
 
 [trainer.optim]

configs/debug/algo/grpo.toml

@@ -17,7 +17,7 @@ group_size = 16
 [orchestrator.train.algo]
 type = "grpo"
 
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"
 
 [orchestrator.train.sampling]

configs/debug/algo/max_rl.toml

@@ -17,7 +17,7 @@ group_size = 16
 [orchestrator.train.algo]
 type = "max_rl"
 
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"
 
 [orchestrator.train.sampling]

configs/debug/algo/mixed_grpo_opd.toml

@@ -27,7 +27,7 @@ group_size = 16
 [orchestrator.train.algo]
 type = "grpo"
 
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"
 
 [orchestrator.train.sampling]

configs/debug/algo/opd.toml

@@ -28,8 +28,8 @@ type = "opd"
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
 base_url = "http://localhost:8001/v1"
 
-# Teacher rolls out over plain chat-completions (no tokens); the renderer backfills them.
-[orchestrator.renderer]
+# The inference server returns exact tokens with the policy chat template.
+[inference.renderer]
 name = "prime-qwen3"
 
 [orchestrator.train.sampling]

configs/debug/algo/opd_lora.toml

@@ -28,8 +28,8 @@ type = "opd"
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
 base_url = "http://localhost:8001/v1"
 
-# Teacher rolls out over plain chat-completions (no tokens); the renderer backfills them.
-[orchestrator.renderer]
+# The inference server returns exact tokens with the policy chat template.
+[inference.renderer]
 name = "prime-qwen3"
 
 [orchestrator.train.sampling]

configs/debug/algo/rae.toml

@@ -17,7 +17,7 @@ group_size = 1
 [orchestrator.train.algo]
 type = "rae"
 
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"
 
 [orchestrator.train.sampling]

configs/debug/algo/self_distill.toml

@@ -25,7 +25,7 @@ group_size = 1
 type = "opsd"
 demo_key = "answer"
 
-[orchestrator.renderer]
+[inference.renderer]
 name = "prime-qwen3"
 
 [orchestrator.train.sampling]

configs/debug/algo/sft_distill.toml

@@ -1,6 +1,6 @@
 # Start the frozen reference server first (on a separate GPU):
 #   CUDA_VISIBLE_DEVICES=1 uv run inference \
-#     --vllm.model PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL \
+#     --vllm.model PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL --renderer.name prime-qwen3 \
 #     --server.port 8001 --vllm.gpu-memory-utilization 0.5 --vllm.enforce-eager
 # Then:
 #   uv run rl @ configs/debug/algo/sft_distill.toml
@@ -29,12 +29,8 @@ type = "sft"
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
 base_url = "http://localhost:8001/v1"
 
-# The student's renderer tokenizes the teacher's messages into the student's
-# token space (backfill). Use `prime-qwen3` — it matches this PrimeIntellect
-# Qwen3 model's chat template token-for-token (single-turn + system), keeping the
-# CE target faithful, without the empty `<think></think>` block the stock `qwen3`
-# renderer injects (which this model's template never emits).
-[orchestrator.renderer]
+# Match the policy chat template. The frozen generation server also owns its renderer.
+[inference.renderer]
 name = "prime-qwen3"
 
 [orchestrator.train.sampling]

configs/debug/algo/sft_distill_lora.toml

@@ -1,6 +1,6 @@
 # Start the frozen reference server first (on a separate GPU):
 #   CUDA_VISIBLE_DEVICES=1 uv run inference \
-#     --vllm.model PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL \
+#     --vllm.model PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL --renderer.name prime-qwen3 \
 #     --server.port 8001 --vllm.gpu-memory-utilization 0.5 --vllm.enforce-eager
 # Then:
 #   uv run rl @ configs/debug/algo/sft_distill_lora.toml
@@ -29,12 +29,8 @@ type = "sft"
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
 base_url = "http://localhost:8001/v1"
 
-# The student's renderer tokenizes the teacher's messages into the student's
-# token space (backfill). Use `prime-qwen3` — it matches this PrimeIntellect
-# Qwen3 model's chat template token-for-token (single-turn + system), keeping the
-# CE target faithful, without the empty `<think></think>` block the stock `qwen3`
-# renderer injects (which this model's template never emits).
-[orchestrator.renderer]
+# Match the policy chat template. The frozen generation server also owns its renderer.
+[inference.renderer]
 name = "prime-qwen3"
 
 [orchestrator.train.sampling]

docs/algorithms.md

@@ -490,7 +490,7 @@ reject_max = 0.05
 
 ## Multi-Turn Trajectories
 
-For multi-turn rollouts (tool use, browser environments, long conversations), `prime-rl` records each LLM request/response as an independent **trajectory step** and merges them at training time using best-effort interleaving — with [renderers](#renderers) as the mechanism that keeps the merge safe by construction.
+For multi-turn rollouts (tool use, browser environments, long conversations), `prime-rl` records each LLM request/response as an independent **trajectory step** and merges them at training time using best-effort interleaving. Verifiers checks the server-returned token IDs before sharing a prefix.
 
 ### Extension Property
 
@@ -518,47 +518,40 @@ steps 4–5: extension holds   → merged into Sample 2
 result: 2 training samples instead of 5
 ```
 
-The orchestrator enforces an **exact prefix invariant**: the prompt at turn $t$ must be the exact concatenation of prior messages exactly as the LLM originally generated them. If turn 2's prompt is `U1, A1', U2` while `A1' ≠ A1`, the orchestrator can't safely merge — either choice produces logprob drift between trainer and inference. Starting a fresh sample is the only correct behavior, so that's what happens.
+Verifiers enforces an **exact prefix invariant** when merging records: the prompt at turn $t$ must be the exact concatenation of prior messages exactly as the LLM originally generated them. If turn 2's prompt is `U1, A1', U2` while `A1' ≠ A1`, the orchestrator can't safely merge — either choice produces logprob drift between trainer and inference. Starting a fresh sample is the only correct behavior, so that's what happens.
 
 ### Renderers
 
-Best-effort interleaving works because the renderer guarantees the exact-prefix invariant *by construction* — it never re-renders prior turns, so it can't lose tokens to chat-template normalization, BPE retokenization drift, or thinking stripping. A renderer turns a model's chat template into a Python object that can:
-
-- `render_ids(messages)` — tokenize messages to ids the inference engine accepts.
-- `parse_response(completion_ids)` — recover structured `(content, reasoning_content, tool_calls)` from sampled ids.
-- `bridge_to_next_turn(prev_prompt_ids, prev_completion_ids, new_messages)` — extend the previous turn's tokens verbatim with the new environment turn, instead of re-rendering history.
-
-When `bridge_to_next_turn` succeeds, the trainer sees the exact token stream the sampler produced; when it can't be proven safe (e.g. the renderer is `DefaultRenderer` and the template's stop sequence is unknown), it returns `None` and the orchestrator falls back to a full re-render — which triggers the new-sample fallback above.
-
-A common source of breakage in the absence of a hand-coded renderer is models like Qwen3 whose chat templates strip past `<think>` blocks across user turns:
-
-```python
-from transformers import AutoTokenizer
-tok = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
-messages = [
-    {"role": "user", "content": "U1"},
-    {"role": "assistant", "content": "<think>R1</think>A1"},
-    {"role": "user", "content": "U2"},
-]
-tok.apply_chat_template(messages[:1], tokenize=False)
-# <|im_start|>user
-# U1<|im_end|>
-
-tok.apply_chat_template(messages, tokenize=False)
-# <|im_start|>user\nU1<|im_end|>\n<|im_start|>assistant\nA1<|im_end|>\n<|im_start|>user\nU2<|im_end|>
-# (the <think>R1</think> from turn 2 is gone)
-```
-
-Hand-coded renderers ship for `qwen3`, `qwen3-vl`, `qwen3.5`, `glm-5`, `glm-4.5`, `minimax-m2`, `deepseek-v3`, `kimi-k2`, `kimi-k2.5`, `nemotron-3`, `gpt-oss`; anything else falls back to `DefaultRenderer` (a generic `apply_chat_template` wrapper). Pick one via:
+Evaluation and training use the same OpenAI-compatible client. Training requests ask the
+inference server for exact prompt and completion token IDs, sampled-token logprobs, and
+training metadata. Each call is a valid training record in its own right. Verifiers merges
+records only when their token prefixes match exactly; a re-rendered history starts a new
+sample with its context masked out.
+
+The inference server uses the configured `renderers` implementation for every Chat
+Completions prompt, and to attribute prompt tokens to messages
+and message bodies. ECHO uses this attribution to select observations, including when
+tokenization changes across turns. Multimodal records keep inline image URLs and exact
+placeholder ranges; the trainer builds image features from those URLs. Native vLLM handles
+sampling, response parsing, tools, reasoning, and streaming for both training and eval.
+`return_training_metadata=true` adds exact token IDs and attribution; logprobs remain optional.
+Sampling masks and
+expert routing travel alongside the tokens for replay in the trainer.
 
 ```toml
-[orchestrator.renderer]
-name = "auto"   # detect from tokenizer; pass an explicit name for fine-tunes
+[inference.renderer]
+name = "auto"   # detect from tokenizer; use an explicit name for fine-tunes
 ```
 
 A renderer outside the `renderers` package loads as a custom renderer: `name = "custom"` with `import_path = "my_module.Class"` (or `path/to/file.py:Class`), see [Custom renderers](training.md#dataset-format).
 
-For the full design rationale (failure modes ruled out, empirical token-identity comparison against `apply_chat_template`, when to write a hand-coded renderer), see [the renderers writeup on the Prime Intellect blog](https://www.primeintellect.ai/blog/renderers) — the canonical reference.
+RL and distillation losses require sampling logprobs. CE-only frozen generation can omit
+logprobs, but its endpoint must still return exact tokens and training metadata.
+Before frozen generation starts, its endpoint's `/v1/tokenizer` vocabulary must match
+the policy tokenizer's token-to-ID mapping. Different vocabularies are rejected; frozen
+tokens are not retokenized for the policy.
+If a frozen endpoint's router does not forward `/v1/tokenizer`, set its `admin_base_url`
+to the inference servers so their vocabularies can be checked directly.
 
 ### Discontinuous Trajectories
 

docs/overview.md

@@ -8,8 +8,8 @@ A `prime-rl` RL run is three cooperating processes:
 
 ![Architecture](assets/architecture.png)
 
-- **Inference** — vLLM-backed server (or fleet) holding the current policy. The orchestrator drives rollouts through the token-in `/inference/v1/generate` route via the [`renderers`](https://github.com/PrimeIntellect-ai/renderers) package (OpenAI-compatible chat/completions routes are also exposed for external clients). We are trying to stay up-to-date with the latest vLLM features, you can read more about the supported features and deployment options in the dedicated [inference documentation](inference.md).
-- **Orchestrator** — Lightweight CPU process that owns the data plane across many [`verifiers`](https://github.com/PrimeIntellect-ai/verifiers) training and eval environments. Each env runs in an isolated subprocess with a variable-size pool of env workers for scalability. The orchestrator drives multi-turn rollouts against the inference fleet (tool use, browsers, sandboxes, long horizons) without re-tokenizing across turns, computes advantages, packs the rollouts into training batches, and relays new weights from trainer to inference.
+- **Inference** — vLLM-backed server (or fleet) holding the current policy. The orchestrator drives rollouts through `/v1/chat/completions`, requesting exact tokens and training attribution from the server. The endpoint's [`renderers`](https://github.com/PrimeIntellect-ai/renderers) configuration controls prompt rendering for both training and eval. See the [inference documentation](inference.md) for supported features and deployment options.
+- **Orchestrator** — Lightweight CPU process that owns the data plane across many [`verifiers`](https://github.com/PrimeIntellect-ai/verifiers) training and eval environments. Each env runs in an isolated subprocess with a variable-size pool of env workers for scalability. The orchestrator drives multi-turn rollouts against the inference fleet (tool use, browsers, sandboxes, long horizons), merges exact token prefixes across turns, computes advantages, packs the rollouts into training batches, and relays new weights from trainer to inference.
 - **Trainer** — FSDP2 process group that consumes packed rollouts and steps the optimizer. We ship optimized custom modeling code for many MoE / dense / VLM families that unlocks advanced trainer parallelism — expert parallelism (EP, with DeepEP kernels) and context parallelism (CP) for long-sequence training — plus selective activation checkpointing, FP8 training on Hopper+, and LoRA. You can read more in the dedicated [training documentation](training.md).
 
 The three processes communicate through configurable transports — by default the trainer↔orchestrator rollout link uses ZMQ (each trainer rank requests a step before the orchestrator sends it, so the trainer only ever holds the batch it is training on), and weight broadcast uses NCCL for synchronous in-memory transfer (falling back to filesystem when LoRA is enabled or no inference server is configured). Swap the rollout link to the local filesystem (`rollout_transport.type = "filesystem"`) if you want rollouts persisted to disk. See [Scaling](scaling.md) for the deployment options.

docs/training.md

@@ -193,7 +193,7 @@ import_path = "my_renderers.glm53_no_thinking.NoThinkingGLM53Renderer"
 clear_thinking = true
 ```
 
-The same config selects the renderer for RL under `[orchestrator.renderer]`. The [`renderers` README](https://github.com/PrimeIntellect-ai/renderers#custom-renderers) shows the class behind this example.
+The same config selects the renderer for RL under `[inference.renderer]`. The [`renderers` README](https://github.com/PrimeIntellect-ai/renderers#custom-renderers) shows the class behind this example.
 
 **Renderer-backed tokenization.** SFT tokenization is renderer-only. The [`renderers`](algorithms.md#renderers) package owns message-to-token conversion and loss attribution end-to-end, so position-dependent chat templates (for example templates that strip past `<think>` blocks across user turns) do not corrupt the loss mask. `[renderer]` defaults to `name = "auto"`; set a typed renderer config only when you need model-specific template controls. Hand-coded renderers ship for Qwen3, Qwen3.5, GLM-5, GLM-4.5, Kimi K2/K2.5, MiniMax M2, DeepSeek V3, Nemotron 3, GPT-OSS, and VLM families such as Qwen3-VL/Qwen3.5.
 

examples/advanced/glm-4.5-air/ipo-ablation/base.toml

@@ -66,10 +66,6 @@ num_output_tokens_weight = 0.1
 num_input_tokens_weight = 0.1
 num_turns_weight = 0.1
 
-[orchestrator.renderer]
-name = "glm-4.5"
-thinking_retention = "all"
-
 [[orchestrator.train.source]]
 name = "scaleswe"
 env.taskset.id = "scaleswe"
@@ -89,6 +85,10 @@ env.agent.timeout.rollout = 3600
 
 [inference]
 
+[inference.renderer]
+name = "glm-4.5"
+thinking_retention = "all"
+
 [inference.vllm]
 gpu_memory_utilization = 0.85
 max_model_len = 131072

examples/advanced/glm-4.5-air/search.toml

@@ -64,7 +64,7 @@ group_size = 16
 initial_inflight = 256
 max_inflight = 512
 
-[orchestrator.renderer]
+[inference.renderer]
 name = "glm-4.5"
 thinking_retention = "all"
 

examples/advanced/glm-4.5-air/swe-budget.toml

@@ -82,7 +82,7 @@ num_output_tokens_weight = 0.0
 num_input_tokens_weight = 0.1
 num_turns_weight = 0.1
 
-[orchestrator.renderer]
+[inference.renderer]
 name = "glm-4.5"
 thinking_retention = "all"
 

examples/advanced/glm-4.5-air/swe.toml

@@ -82,7 +82,7 @@ num_output_tokens_weight = 0.0
 num_input_tokens_weight = 0.1
 num_turns_weight = 0.1
 
-[orchestrator.renderer]
+[inference.renderer]
 name = "glm-4.5"
 thinking_retention = "all"
 

examples/advanced/glm-4.5-air/terminal.toml

@@ -65,7 +65,7 @@ group_size = 16
 initial_inflight = 256
 max_inflight = 512
 
-[orchestrator.renderer]
+[inference.renderer]
 name = "glm-4.5"
 thinking_retention = "all"