Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@

import torch
import torch.distributed as dist
from megatron.core.utils import unwrap_model

from miles.backends.training_utils.parallel import get_parallel_state
from miles.backends.training_utils.weight_update.hf_weight_iterator import HfWeightIteratorBase, WeightUpdatePlacement
Expand All @@ -35,9 +34,6 @@ def __init__(self, *args, **kwargs):
and self.quantization_config["quant_method"] == "compressed-tensors"
else None
)
trainer_has_mtp = bool(unwrap_model(self.model)[0].config.mtp_num_layers)
if self.args.sglang_speculative_algorithm and not trainer_has_mtp:
self.weight_update_selector = "target"

def _hf_atomic_update_groups(self):
return get_hf_atomic_update_groups(self.model_name, q_lora_rank=self.args.q_lora_rank)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,6 @@ class HfWeightIteratorBase(ABC):

# Placement this implementation can produce; None means any.
forced_placement: ClassVar[WeightUpdatePlacement | None] = None
# Engine runners a sync covers ("all" or "target"); a backend narrows it when its draft is frozen.
weight_update_selector: str = "all"

def __init__(
self,
Expand Down
8 changes: 8 additions & 0 deletions miles/backends/training_utils/weight_update/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,14 @@ def resume_engines(rollout_engines: Sequence[SGLangApiClient]) -> None:
async_utils.wait_futures([async_utils.submit(client.continue_generation()) for client in rollout_engines])


def weight_update_selector(args: Namespace) -> str:
"""Engine runners a weight sync covers. A speculative draft is synced only when the
trainer trains its MTP layers (--enable-mtp-training); any other draft keeps its startup weights."""
if args.sglang_speculative_algorithm and not args.enable_mtp_training:
return "target"
return "all"


def begin_weight_update(
rollout_engines: Sequence[SGLangApiClient], selector: str = "all", *, sync_base: bool = True
) -> None:
Expand Down
7 changes: 3 additions & 4 deletions miles/backends/training_utils/weight_update/updater.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
register_lora_adapter,
resume_engines,
set_weight_version,
weight_update_selector,
)
from miles.backends.training_utils.weight_update.utils import record_lora_checksums
from miles.utils import async_utils
Expand Down Expand Up @@ -88,7 +89,7 @@ def connect_rollout_engines(
engine_gpu_offsets,
self.parallel_state,
self._hf_weight_iterator.placement,
self._hf_weight_iterator.weight_update_selector,
weight_update_selector(self.args),
)
assert self.protocol.is_sender is not None, "connect() must set is_sender"
self._registered_adapters.clear()
Expand Down Expand Up @@ -135,9 +136,7 @@ def update_weights(self) -> None:
if protocol.use_weight_update_session and driver:
pause_engines(self.args, protocol.rollout_engines)
self._register_new_lora_adapters(protocol.rollout_engines, adapters)
begin_weight_update(
protocol.rollout_engines, self._hf_weight_iterator.weight_update_selector, sync_base=sync_base
)
begin_weight_update(protocol.rollout_engines, weight_update_selector(self.args), sync_base=sync_base)
dist.barrier(group=get_gloo_group())

checksums = {name: {} for name, _ in adapters} if self.is_lora and self.args.check_lora_weight_equal else None
Expand Down
9 changes: 7 additions & 2 deletions miles/ray/rollout/server_cell.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,12 @@ async def _tick_when_initializing(self) -> None:
return

if self.args.check_weight_update_equal and self.meta.update_weights:
await self.check_weights(action="snapshot", allow_quant_error=False, selector="all", skip_list=None)
await self.check_weights(
action="snapshot",
allow_quant_error=False,
selector=self.args.check_weight_update_selector,
skip_list=None,
)

if self.meta.needs_offload:
api_client = SGLangApiClient(server_url=addr_info.server_url)
Expand All @@ -191,7 +196,7 @@ async def _tick_when_initializing(self) -> None:
await self.check_weights(
action="reset_tensors",
allow_quant_error=False,
selector="all",
selector=self.args.check_weight_update_selector,
skip_list=self.args.check_weight_update_skip_list,
)

Expand Down
13 changes: 9 additions & 4 deletions miles/utils/arguments.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
)
from miles.backends.sglang_utils.arguments import add_sglang_arguments, collect_eval_sglang_overrides
from miles.backends.sglang_utils.arguments import validate_args as sglang_validate_args
from miles.backends.training_utils.weight_update.session import weight_update_selector
from miles.dashboard.args import add_dashboard_arguments, validate_dashboard_args
from miles.ray.specs.train import compute_trainer_ids, external_trainer_controller_addrs
from miles.rollout.checkpoint_eval import is_checkpoint_eval_fn
Expand Down Expand Up @@ -2247,11 +2248,11 @@ def add_debug_arguments(parser):
parser.add_argument(
"--check-weight-update-selector",
type=str,
default="all",
default=None,
choices=["all", "target", "draft"],
help="Which model the post-update equality check covers: 'all' (target + "
"draft/MTP), 'target' (target model only; skips the draft, e.g. when MTP "
"training is off), or 'draft' (draft/MTP worker only).",
help="Which engine runners the post-update equality check snapshots, resets and compares: "
"'all' (target + draft/MTP), 'target' or 'draft'. Defaults to the runners the weight sync "
"covers, which leaves out a speculative draft unless --enable-mtp-training trains it.",
)
parser.add_argument(
"--check-weight-update-skip-list",
Expand Down Expand Up @@ -3612,6 +3613,10 @@ def miles_validate_args(args):
):
args.check_weight_update_equal = True

if args.check_weight_update_selector is None:
# reset_tensors scrambles every tensor it covers, so it must cover only what the sync rewrites
args.check_weight_update_selector = weight_update_selector(args)

# always true on offload for colocate at the moment.
if args.update_weight_transfer_mode == "p2p":
assert not args.colocate, (
Expand Down
2 changes: 0 additions & 2 deletions tests/e2e/megatron/test_glm47_flash/_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,8 +263,6 @@ def build_train_args(case: CaseConfig, *, wandb_file: str) -> str:
mtp_args = "--mtp-num-layers 0 "

ci_args = "--ci-test "
if not case.use_spec:
ci_args += "--check-weight-update-selector target "
if case.use_fp8_rollout:
ci_args += "--check-weight-update-allow-quant-error "

Expand Down
11 changes: 3 additions & 8 deletions tests/e2e/megatron/test_qwen3_5_35B_A3B/_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,6 @@ class CaseConfig:
# Whether to enable R3 routing replay (--use-rollout-routing-replay).
use_r3: bool
max_tokens_per_gpu: int = 8192
# Weight-check selector: "all" (target + draft) or "target" (target model only; use
# when MTP training is off so the un-synced draft is not checked).
check_weight_update_selector: str = "all"
# Rollout weight-name substrings to exclude from the equality check (substring match;
# mismatches become non-fatal). Cases pass ("visual",): miles has no VLM/vision
# implementation on the training side, so those weights are never synced.
Expand Down Expand Up @@ -82,8 +79,8 @@ def __post_init__(self):
assert self.update_weight_transfer_mode in ("broadcast", "p2p")
if self.update_weight_transfer_mode == "p2p" and self.colocate:
raise ValueError("p2p weight updates require colocate=False: the engines are written from other GPUs")
if not self.use_spec and (self.enable_mtp_training or self.check_weight_update_selector != "target"):
raise ValueError("without spec there is no draft: set enable_mtp_training=False and selector 'target'")
if not self.use_spec and self.enable_mtp_training:
raise ValueError("without spec there is no draft to train: set enable_mtp_training=False")
if self.sglang_deepep_dispatcher_output_dtype is not None and not self.use_deepep:
raise ValueError("sglang_deepep_dispatcher_output_dtype requires use_deepep=True")

Expand Down Expand Up @@ -226,8 +223,7 @@ def build_train_args(case: CaseConfig, *, wandb_file: str) -> str:
sglang_args += f"--sglang-deepep-dispatcher-output-dtype {case.sglang_deepep_dispatcher_output_dtype} "

# When MTP training is off the rollout still runs EAGLE spec from the checkpoint
# draft; those draft weights just never get synced (the spec_nor3 case checks only the
# target through its selector).
# draft; the weight sync and the weight check both leave that draft out.
if case.enable_mtp_training:
mtp_args = "--enable-mtp-training " "--mtp-num-layers 1 " "--mtp-loss-scaling-factor 0.2 "
else:
Expand All @@ -236,7 +232,6 @@ def build_train_args(case: CaseConfig, *, wandb_file: str) -> str:
mtp_args = "--mtp-num-layers 0 "

ci_args = "--ci-test "
ci_args += f"--check-weight-update-selector {case.check_weight_update_selector} "
if case.check_weight_update_skip_list:
ci_args += "--check-weight-update-skip-list " + " ".join(case.check_weight_update_skip_list) + " "
if case.use_fp8_rollout:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
use_spec=False,
enable_mtp_training=False,
use_r3=False,
check_weight_update_selector="target",
# miles has no VLM/vision implementation on the training side, so vision weights are
# never synced; exclude them from the weight-equality check.
check_weight_update_skip_list=("visual",),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
use_spec=False,
enable_mtp_training=False,
use_r3=False,
check_weight_update_selector="target",
# miles has no VLM/vision implementation on the training side, so vision weights are
# never synced; exclude them from the weight-equality check.
check_weight_update_skip_list=("visual",),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@
use_spec=False,
enable_mtp_training=False,
use_r3=True,
check_weight_update_selector="target",
# miles has no VLM/vision implementation on the training side, so vision weights are
# never synced; exclude them from the weight-equality check.
check_weight_update_skip_list=("visual",),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@
use_fp8_rollout=True,
enable_mtp_training=False,
use_r3=False,
check_weight_update_selector="target",
# miles has no VLM/vision implementation on the training side, so vision weights are
# never synced; exclude them from the weight-equality check.
check_weight_update_skip_list=("visual",),
Expand Down
45 changes: 0 additions & 45 deletions tests/fast/backends/megatron_utils/test_weight_update_selector.py

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,12 @@ def _make_updater(engines: list[_RecordingApiClient], *, pause_generation_mode:
)
iterator = MagicMock()
iterator.iter_hf_weights.return_value = iter([])
iterator.weight_update_selector = "all"
args = Namespace(pause_generation_mode=pause_generation_mode, check_lora_weight_equal=False)
args = Namespace(
pause_generation_mode=pause_generation_mode,
check_lora_weight_equal=False,
sglang_speculative_algorithm=None,
enable_mtp_training=False,
)
with patch(f"{_UPDATER_MODULE}.get_weight_transfer_protocol", return_value=protocol):
return WeightUpdater(
args,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ def _make_args(**overrides):
actor_num_nodes=1,
actor_num_gpus_per_node=1,
pause_generation_mode="retract",
sglang_speculative_algorithm=None,
enable_mtp_training=False,
)
defaults.update(overrides)
return Namespace(**defaults)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
"""The weight-update selector follows whether the trainer trains MTP layers, on every backend."""

from tests.ci.ci_register import register_cpu_ci

register_cpu_ci(est_time=60, suite="stage-b-cpu", labels=["weight-update"])

from argparse import Namespace

import pytest

from miles.backends.training_utils.weight_update.session import weight_update_selector


@pytest.mark.parametrize(
"speculative, enable_mtp_training, selector",
[
(None, False, "all"),
(None, True, "all"),
("EAGLE", True, "all"),
("EAGLE", False, "target"),
("EAGLE3", False, "target"),
],
ids=["no_draft", "no_draft_with_mtp", "trained_draft", "frozen_draft", "external_draft"],
)
def test_a_draft_is_synced_only_when_the_trainer_trains_mtp_layers(speculative, enable_mtp_training, selector):
"""FSDP and torchtitan reject --enable-mtp-training, so their drafts are always left out."""
args = Namespace(sglang_speculative_algorithm=speculative, enable_mtp_training=enable_mtp_training)
assert weight_update_selector(args) == selector
1 change: 1 addition & 0 deletions tests/fast/ray/rollout/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ def make_args(**overrides: Any) -> Namespace:
update_weights_interval=1,
check_weight_update_equal=False,
check_weight_update_skip_list=None,
check_weight_update_selector="all",
# batch / training
global_batch_size=8,
use_dynamic_global_batch_size=False,
Expand Down
13 changes: 13 additions & 0 deletions tests/fast/ray/rollout/test_server_cell_state_machine.py
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,19 @@ async def test_a_cell_primes_the_weight_update_checker_before_its_first_update(s
assert [kwargs["action"] for kwargs in checker_calls] == ["snapshot", "reset_tensors"]
assert checker_calls[1]["skip_list"] == ["a"]

async def test_the_checker_snapshots_and_resets_only_the_runners_it_compares(self, cell_env):
"""A draft the sync leaves out would stay scrambled for the rest of the run."""
cell = _make_cell(args_overrides=dict(check_weight_update_equal=True, check_weight_update_selector="target"))
await cell.init()

await cell.tick()

checker_calls = [kwargs for name, kwargs in cell_env["memory_calls"] if name == "check_weights"]
assert [(kwargs["action"], kwargs["selector"]) for kwargs in checker_calls] == [
("snapshot", "target"),
("reset_tensors", "target"),
]

async def test_the_weight_update_checker_stays_out_of_the_way_when_it_is_disabled(self, cell_env):
"""The checker is a ci-only tool and must not touch engines in a normal run."""
cell = _make_cell()
Expand Down
25 changes: 25 additions & 0 deletions tests/fast/utils/test_arguments.py
Original file line number Diff line number Diff line change
Expand Up @@ -3078,3 +3078,28 @@ def test_fully_async_without_over_sampling_batch_size_does_not_warn(caplog):
_resolve_rollout_functions(args)

assert "over-sampling-batch-size" not in caplog.text


class TestCheckWeightUpdateSelectorDefault:
"""reset_tensors scrambles every tensor the checker covers, so by default it covers what the sync rewrites."""

def _validate(self, extra):
parser = argparse.ArgumentParser()
get_miles_extra_args_provider()(parser)
args = parser.parse_args(extra + ["--num-rollout", "1"] + REQUIRED_ARGS)
_set_megatron_parallel_sizes(args)
miles_validate_args(args)
return args

@pytest.mark.parametrize(
("extra", "selector"),
[
([], "all"),
(["--sglang-speculative-algorithm", "EAGLE"], "target"),
(["--sglang-speculative-algorithm", "EAGLE", "--enable-mtp-training", "--mtp-num-layers", "1"], "all"),
(["--sglang-speculative-algorithm", "EAGLE", "--check-weight-update-selector", "draft"], "draft"),
],
ids=["no_draft", "frozen_draft", "trained_draft", "explicit"],
)
def test_the_checker_covers_what_the_sync_rewrites_unless_narrowed(self, extra, selector):
assert self._validate(extra).check_weight_update_selector == selector
Loading