Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/user-guide/rollout-endpoints.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,14 +86,14 @@ Helpers:

- `compute_prompt_ids_from_sample` and `compute_request_payload` from
`miles/rollout/generate_utils/generate_endpoint_utils.py` build `/generate` requests.
- For multi-sample outputs, set `--generate-multi-samples` and return a list.
- A generate function can set `GenerateFnOutput.samples` to a `Sample` or `list[Sample]`.

### Reference generators

- **`single_turn.py`**: single-turn generation via `/generate`. Text or multimodal prompts.
- **`multi_turn.py`**: multi-turn tool calling via `/generate`. Adds CLI flags
`--generate-max-turns`, `--generate-tool-specs-path`, `--generate-tool-call-parser`,
`--generate-execute-tool-function-path`, `--generate-multi-samples`.
`--generate-execute-tool-function-path`.
- **`benchmarkers.py`**: forces random output sequence length for benchmarking.

---
Expand Down
24 changes: 12 additions & 12 deletions miles/rollout/filter_hub/dynamic_sampling_filters.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,25 +6,25 @@
__all__ = ["check_reward_nonzero_std", "check_no_aborted"]


def check_reward_nonzero_std(args, samples: list[Sample], **kwargs):
rewards = [sample.get_reward_value(args) for sample in samples]
keep = torch.tensor(rewards, dtype=torch.float64).std() > 1e-8
return DynamicFilterOutput(
keep=keep,
reason=None if keep else f"zero_std_{round(rewards[0], 1)}",
)


def _flatten_samples(samples):
"""Flatten samples that may contain nested lists (from --generate-multi-samples)."""
def _flatten_samples(samples: list[Sample | list[Sample]]):
"""Flatten a group whose elements are `Sample` or `list[Sample]` (generate-function dependent)."""
for s in samples:
if isinstance(s, list):
yield from s
else:
yield s


def check_no_aborted(args, samples: list[Sample], **kwargs):
def check_reward_nonzero_std(args, samples: list[Sample | list[Sample]], **kwargs):
rewards = [sample.get_reward_value(args) for sample in _flatten_samples(samples)]
keep = torch.tensor(rewards, dtype=torch.float64).std() > 1e-8
return DynamicFilterOutput(
keep=keep,
reason=None if keep else f"zero_std_{round(rewards[0], 1)}",
)


def check_no_aborted(args, samples: list[Sample | list[Sample]], **kwargs):
"""Reject entire group if any sample was aborted (e.g. env timeout, Docker crash)."""
if any(s.status == Sample.Status.ABORTED for s in _flatten_samples(samples)):
return DynamicFilterOutput(keep=False, reason="group_has_aborted")
Expand Down
10 changes: 3 additions & 7 deletions miles/rollout/generate_hub/agentic_tool_call.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,17 +125,13 @@ async def generate(input: GenerateFnInput) -> GenerateFnOutput:
sample.status = Sample.Status.ABORTED
return GenerateFnOutput(samples=sample)

if not input.args.generate_multi_samples:
samples = merge_samples(samples, input.state.tokenizer)
samples.metadata.update(session_metadata)
else:
samples[-1].metadata.update(session_metadata)
return GenerateFnOutput(samples=samples)
sample = merge_samples(samples, input.state.tokenizer)
sample.metadata.update(session_metadata)
return GenerateFnOutput(samples=sample)


def _add_arguments(parser: argparse.ArgumentParser):
parser.add_argument("--custom-agent-function-path", type=str)
parser.add_argument("--generate-multi-samples", action="store_true", default=False)
parser.add_argument(
"--max-seq-len",
type=int,
Expand Down
15 changes: 2 additions & 13 deletions miles/rollout/generate_hub/multi_turn.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,6 @@ async def generate(input: GenerateFnInput) -> GenerateFnOutput:
tool_specs = load_function(args.generate_tool_specs_path)
tool_call_parser = create_tool_call_parser(tool_specs, args.generate_tool_call_parser)

multi_samples = []

# tool-call markup stays inline in the assistant text (what the model emitted)
record_trajectory = args.save_debug_trajectory_data is not None
trajectory = (list(sample.prompt) if isinstance(sample.prompt, list) else []) if record_trajectory else None
Expand All @@ -56,13 +54,8 @@ async def generate(input: GenerateFnInput) -> GenerateFnOutput:
payload, halt_status = compute_request_payload(args, sample.tokens, input.sampling_params)
if payload is None:
sample.status = halt_status
if args.generate_multi_samples and multi_samples:
multi_samples[-1].status = halt_status
break

if args.generate_multi_samples:
sample = deepcopy(input.sample)

gen_t0 = time.time()
output = await post(url, payload, headers=compute_routing_headers(args, sample))
sink = None if input.evaluation else TrajectoryLifecycle().sink
Expand All @@ -72,10 +65,7 @@ async def generate(input: GenerateFnInput) -> GenerateFnOutput:
await update_sample_from_response(args, sample, payload=payload, output=output, update_loss_mask=True)
if record_trajectory:
trajectory.append({"role": "assistant", "content": output["text"]})
sample.metadata["messages"] = list(trajectory) # snapshot: multi_samples deepcopies per turn

if args.generate_multi_samples:
multi_samples.append(deepcopy(sample))
sample.metadata["messages"] = list(trajectory) # snapshot: trajectory keeps growing in later turns

if output["meta_info"]["finish_reason"]["type"] in ("abort", "length"):
break
Expand All @@ -95,15 +85,14 @@ async def generate(input: GenerateFnInput) -> GenerateFnOutput:
trajectory.extend(tool_messages)
sample.metadata["messages"] = list(trajectory)

return GenerateFnOutput(samples=multi_samples if args.generate_multi_samples else sample)
return GenerateFnOutput(samples=sample)


def _add_arguments(parser: argparse.ArgumentParser):
parser.add_argument("--generate-max-turns", type=int, default=16)
parser.add_argument("--generate-tool-specs-path", type=str)
parser.add_argument("--generate-tool-call-parser", type=str)
parser.add_argument("--generate-execute-tool-function-path", type=str)
parser.add_argument("--generate-multi-samples", action="store_true")


generate.add_arguments = _add_arguments
14 changes: 4 additions & 10 deletions tests/fast/fixtures/generation_fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,8 @@
VARIANT_TO_GENERATE_FN_PATH = {
"old_sglang_rollout": "miles.rollout.sglang_rollout.generate",
"single_turn": "miles.rollout.generate_hub.single_turn.generate",
"multi_turn_single_sample": "miles.rollout.generate_hub.multi_turn.generate",
"multi_turn_multi_samples": "miles.rollout.generate_hub.multi_turn.generate",
"agentic_tool_call_single_sample": "miles.rollout.generate_hub.agentic_tool_call.generate",
"agentic_tool_call_multi_samples": "miles.rollout.generate_hub.agentic_tool_call.generate",
"multi_turn": "miles.rollout.generate_hub.multi_turn.generate",
"agentic_tool_call": "miles.rollout.generate_hub.agentic_tool_call.generate",
}


Expand All @@ -53,7 +51,7 @@ def extra_argv_for_variant(
custom_generate_function_path or VARIANT_TO_GENERATE_FN_PATH[variant],
]

if variant in ("multi_turn_single_sample", "multi_turn_multi_samples"):
if variant == "multi_turn":
argv += [
"--generate-max-turns",
str(generate_max_turns),
Expand All @@ -63,13 +61,9 @@ def extra_argv_for_variant(
generate_execute_tool_function_path,
]
argv += ["--generate-tool-call-parser", generate_tool_call_parser]
if variant == "multi_turn_multi_samples":
argv.append("--generate-multi-samples")
elif variant in ("agentic_tool_call_single_sample", "agentic_tool_call_multi_samples"):
elif variant == "agentic_tool_call":
argv += ["--custom-agent-function-path", custom_agent_function_path]
argv += ["--use-session-server", "--tito-model", "qwen3"]
if variant == "agentic_tool_call_multi_samples":
argv.append("--generate-multi-samples")

return argv

Expand Down
Loading
Loading