diff --git a/miles/backends/megatron_utils/update_weight/hf_weight_iterator.py b/miles/backends/megatron_utils/update_weight/hf_weight_iterator.py index 4fa25c19fb1..8f0adf61601 100644 --- a/miles/backends/megatron_utils/update_weight/hf_weight_iterator.py +++ b/miles/backends/megatron_utils/update_weight/hf_weight_iterator.py @@ -11,7 +11,6 @@ import torch import torch.distributed as dist -from megatron.core.utils import unwrap_model from miles.backends.training_utils.parallel import get_parallel_state from miles.backends.training_utils.weight_update.hf_weight_iterator import HfWeightIteratorBase, WeightUpdatePlacement @@ -35,9 +34,6 @@ def __init__(self, *args, **kwargs): and self.quantization_config["quant_method"] == "compressed-tensors" else None ) - trainer_has_mtp = bool(unwrap_model(self.model)[0].config.mtp_num_layers) - if self.args.sglang_speculative_algorithm and not trainer_has_mtp: - self.weight_update_selector = "target" def _hf_atomic_update_groups(self): return get_hf_atomic_update_groups(self.model_name, q_lora_rank=self.args.q_lora_rank) diff --git a/miles/backends/training_utils/weight_update/hf_weight_iterator/__init__.py b/miles/backends/training_utils/weight_update/hf_weight_iterator/__init__.py index ef9c4595fa0..d1de046954b 100644 --- a/miles/backends/training_utils/weight_update/hf_weight_iterator/__init__.py +++ b/miles/backends/training_utils/weight_update/hf_weight_iterator/__init__.py @@ -57,8 +57,6 @@ class HfWeightIteratorBase(ABC): # Placement this implementation can produce; None means any. forced_placement: ClassVar[WeightUpdatePlacement | None] = None - # Engine runners a sync covers ("all" or "target"); a backend narrows it when its draft is frozen. - weight_update_selector: str = "all" def __init__( self, diff --git a/miles/backends/training_utils/weight_update/session.py b/miles/backends/training_utils/weight_update/session.py index b7efb84e579..dd631878bdf 100644 --- a/miles/backends/training_utils/weight_update/session.py +++ b/miles/backends/training_utils/weight_update/session.py @@ -27,6 +27,14 @@ def resume_engines(rollout_engines: Sequence[SGLangApiClient]) -> None: async_utils.wait_futures([async_utils.submit(client.continue_generation()) for client in rollout_engines]) +def weight_update_selector(args: Namespace) -> str: + """Engine runners a weight sync covers. A speculative draft is synced only when the + trainer trains its MTP layers (--enable-mtp-training); any other draft keeps its startup weights.""" + if args.sglang_speculative_algorithm and not args.enable_mtp_training: + return "target" + return "all" + + def begin_weight_update( rollout_engines: Sequence[SGLangApiClient], selector: str = "all", *, sync_base: bool = True ) -> None: diff --git a/miles/backends/training_utils/weight_update/updater.py b/miles/backends/training_utils/weight_update/updater.py index ac4c65048aa..481bdc5096f 100644 --- a/miles/backends/training_utils/weight_update/updater.py +++ b/miles/backends/training_utils/weight_update/updater.py @@ -27,6 +27,7 @@ register_lora_adapter, resume_engines, set_weight_version, + weight_update_selector, ) from miles.backends.training_utils.weight_update.utils import record_lora_checksums from miles.utils import async_utils @@ -88,7 +89,7 @@ def connect_rollout_engines( engine_gpu_offsets, self.parallel_state, self._hf_weight_iterator.placement, - self._hf_weight_iterator.weight_update_selector, + weight_update_selector(self.args), ) assert self.protocol.is_sender is not None, "connect() must set is_sender" self._registered_adapters.clear() @@ -135,9 +136,7 @@ def update_weights(self) -> None: if protocol.use_weight_update_session and driver: pause_engines(self.args, protocol.rollout_engines) self._register_new_lora_adapters(protocol.rollout_engines, adapters) - begin_weight_update( - protocol.rollout_engines, self._hf_weight_iterator.weight_update_selector, sync_base=sync_base - ) + begin_weight_update(protocol.rollout_engines, weight_update_selector(self.args), sync_base=sync_base) dist.barrier(group=get_gloo_group()) checksums = {name: {} for name, _ in adapters} if self.is_lora and self.args.check_lora_weight_equal else None diff --git a/miles/ray/rollout/server_cell.py b/miles/ray/rollout/server_cell.py index c8df0f0ac5d..8df2077e388 100644 --- a/miles/ray/rollout/server_cell.py +++ b/miles/ray/rollout/server_cell.py @@ -179,7 +179,12 @@ async def _tick_when_initializing(self) -> None: return if self.args.check_weight_update_equal and self.meta.update_weights: - await self.check_weights(action="snapshot", allow_quant_error=False, selector="all", skip_list=None) + await self.check_weights( + action="snapshot", + allow_quant_error=False, + selector=self.args.check_weight_update_selector, + skip_list=None, + ) if self.meta.needs_offload: api_client = SGLangApiClient(server_url=addr_info.server_url) @@ -191,7 +196,7 @@ async def _tick_when_initializing(self) -> None: await self.check_weights( action="reset_tensors", allow_quant_error=False, - selector="all", + selector=self.args.check_weight_update_selector, skip_list=self.args.check_weight_update_skip_list, ) diff --git a/miles/utils/arguments.py b/miles/utils/arguments.py index 1bf4b0d7b64..06f598de86e 100644 --- a/miles/utils/arguments.py +++ b/miles/utils/arguments.py @@ -18,6 +18,7 @@ ) from miles.backends.sglang_utils.arguments import add_sglang_arguments, collect_eval_sglang_overrides from miles.backends.sglang_utils.arguments import validate_args as sglang_validate_args +from miles.backends.training_utils.weight_update.session import weight_update_selector from miles.dashboard.args import add_dashboard_arguments, validate_dashboard_args from miles.ray.specs.train import compute_trainer_ids, external_trainer_controller_addrs from miles.rollout.checkpoint_eval import is_checkpoint_eval_fn @@ -2248,11 +2249,11 @@ def add_debug_arguments(parser): parser.add_argument( "--check-weight-update-selector", type=str, - default="all", + default=None, choices=["all", "target", "draft"], - help="Which model the post-update equality check covers: 'all' (target + " - "draft/MTP), 'target' (target model only; skips the draft, e.g. when MTP " - "training is off), or 'draft' (draft/MTP worker only).", + help="Which engine runners the post-update equality check snapshots, resets and compares: " + "'all' (target + draft/MTP), 'target' or 'draft'. Defaults to the runners the weight sync " + "covers, which leaves out a speculative draft unless --enable-mtp-training trains it.", ) parser.add_argument( "--check-weight-update-skip-list", @@ -3613,6 +3614,10 @@ def miles_validate_args(args): ): args.check_weight_update_equal = True + if args.check_weight_update_selector is None: + # reset_tensors scrambles every tensor it covers, so it must cover only what the sync rewrites + args.check_weight_update_selector = weight_update_selector(args) + # always true on offload for colocate at the moment. if args.update_weight_transfer_mode == "p2p": assert not args.colocate, ( diff --git a/tests/e2e/megatron/test_glm47_flash/_common.py b/tests/e2e/megatron/test_glm47_flash/_common.py index bc549cb8977..b02ca398998 100644 --- a/tests/e2e/megatron/test_glm47_flash/_common.py +++ b/tests/e2e/megatron/test_glm47_flash/_common.py @@ -260,8 +260,6 @@ def build_train_args(case: CaseConfig, *, wandb_file: str) -> str: mtp_args = "--enable-mtp-training " "--mtp-loss-scaling-factor 0.2 " ci_args = "--ci-test " - if not case.use_spec: - ci_args += "--check-weight-update-selector target " if case.use_fp8_rollout: ci_args += "--check-weight-update-allow-quant-error " diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/_common.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/_common.py index ff125fab2b5..90ec95ad84b 100644 --- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/_common.py +++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/_common.py @@ -42,9 +42,6 @@ class CaseConfig: # Whether to enable R3 routing replay (--use-rollout-routing-replay). use_r3: bool max_tokens_per_gpu: int = 8192 - # Weight-check selector: "all" (target + draft) or "target" (target model only; use - # when MTP training is off so the un-synced draft is not checked). - check_weight_update_selector: str = "all" # Rollout weight-name substrings to exclude from the equality check (substring match; # mismatches become non-fatal). Cases pass ("visual",): miles has no VLM/vision # implementation on the training side, so those weights are never synced. @@ -82,8 +79,8 @@ def __post_init__(self): assert self.update_weight_transfer_mode in ("broadcast", "p2p") if self.update_weight_transfer_mode == "p2p" and self.colocate: raise ValueError("p2p weight updates require colocate=False: the engines are written from other GPUs") - if not self.use_spec and (self.enable_mtp_training or self.check_weight_update_selector != "target"): - raise ValueError("without spec there is no draft: set enable_mtp_training=False and selector 'target'") + if not self.use_spec and self.enable_mtp_training: + raise ValueError("without spec there is no draft to train: set enable_mtp_training=False") if self.sglang_deepep_dispatcher_output_dtype is not None and not self.use_deepep: raise ValueError("sglang_deepep_dispatcher_output_dtype requires use_deepep=True") @@ -226,14 +223,12 @@ def build_train_args(case: CaseConfig, *, wandb_file: str) -> str: sglang_args += f"--sglang-deepep-dispatcher-output-dtype {case.sglang_deepep_dispatcher_output_dtype} " # When MTP training is off the rollout still runs EAGLE spec from the checkpoint - # draft; those draft weights just never get synced (the spec_nor3 case checks only the - # target through its selector). + # draft; the weight sync and the weight check both leave that draft out. mtp_args = "" if case.enable_mtp_training: mtp_args = "--enable-mtp-training " "--mtp-num-layers 1 " "--mtp-loss-scaling-factor 0.2 " ci_args = "--ci-test " - ci_args += f"--check-weight-update-selector {case.check_weight_update_selector} " if case.check_weight_update_skip_list: ci_args += "--check-weight-update-skip-list " + " ".join(case.check_weight_update_skip_list) + " " if case.use_fp8_rollout: diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py index 5da450cda26..ed072c42a9f 100644 --- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py +++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py @@ -42,7 +42,6 @@ use_spec=False, enable_mtp_training=False, use_r3=False, - check_weight_update_selector="target", # miles has no VLM/vision implementation on the training side, so vision weights are # never synced; exclude them from the weight-equality check. check_weight_update_skip_list=("visual",), diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py index 03716e12422..34f3835519b 100644 --- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py +++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py @@ -41,7 +41,6 @@ use_spec=False, enable_mtp_training=False, use_r3=False, - check_weight_update_selector="target", # miles has no VLM/vision implementation on the training side, so vision weights are # never synced; exclude them from the weight-equality check. check_weight_update_skip_list=("visual",), diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_r3_bf16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_r3_bf16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py index 82e3d32975a..127b1675ca7 100644 --- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_r3_bf16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py +++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_r3_bf16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py @@ -50,7 +50,6 @@ use_spec=False, enable_mtp_training=False, use_r3=True, - check_weight_update_selector="target", # miles has no VLM/vision implementation on the training side, so vision weights are # never synced; exclude them from the weight-equality check. check_weight_update_skip_list=("visual",), diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py index 1e57d609738..45c2c71d74e 100644 --- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py +++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py @@ -50,7 +50,6 @@ use_fp8_rollout=True, enable_mtp_training=False, use_r3=False, - check_weight_update_selector="target", # miles has no VLM/vision implementation on the training side, so vision weights are # never synced; exclude them from the weight-equality check. check_weight_update_skip_list=("visual",), diff --git a/tests/fast/backends/megatron_utils/test_megatron_config.py b/tests/fast/backends/megatron_utils/test_megatron_config.py index 9043eba96eb..0cb8df654f2 100644 --- a/tests/fast/backends/megatron_utils/test_megatron_config.py +++ b/tests/fast/backends/megatron_utils/test_megatron_config.py @@ -686,6 +686,23 @@ def test_a_tokenizer_named_on_the_command_line_is_left_alone(self, tmp_path): assert _model_args(args, model_id="a").tokenizer_model == "/models/shared" + @pytest.mark.parametrize(("enable_mtp_training", "expected_layers"), [(False, None), (True, 1)]) + def test_a_policy_builds_its_own_mtp_layers_only_when_the_run_trains_mtp( + self, tmp_path, enable_mtp_training, expected_layers + ): + """The override lands after validation, yet Megatron would still train every MTP layer it builds.""" + path = _write_yaml({"trainers": [{"model_id": "a", "overrides": {"mtp_num_layers": 1}}]}, tmp_path) + args = _make_args(path, enable_mtp_training=enable_mtp_training, mtp_num_layers=expected_layers) + + assert _model_args(args, model_id="a").mtp_num_layers == expected_layers + + def test_a_policy_cannot_drop_the_mtp_layers_the_run_trains(self, tmp_path): + """--enable-mtp-training is run-wide, so every policy has to build the MTP layers it trains.""" + path = _write_yaml({"trainers": [{"model_id": "a", "overrides": {"mtp_num_layers": 0}}]}, tmp_path) + + with pytest.raises(AssertionError, match="needs --mtp-num-layers > 0"): + _model_args(_make_args(path, enable_mtp_training=True, mtp_num_layers=1), model_id="a") + class TestMultiPolicyIds: def test_an_explicitly_named_policy_keeps_its_trainer_model_id(self, tmp_path): diff --git a/tests/fast/backends/megatron_utils/test_weight_update_selector.py b/tests/fast/backends/megatron_utils/test_weight_update_selector.py deleted file mode 100644 index 878c2085fc0..00000000000 --- a/tests/fast/backends/megatron_utils/test_weight_update_selector.py +++ /dev/null @@ -1,45 +0,0 @@ -"""The weight-update selector follows the trainer model, not the conversion mode.""" - -from tests.ci.ci_register import register_cpu_ci - -register_cpu_ci(est_time=60, suite="stage-b-cpu", labels=["weight-update"]) - -from argparse import Namespace -from types import SimpleNamespace - -import pytest - -from miles.backends.megatron_utils.update_weight.hf_weight_iterator import MegatronHfWeightIteratorBase -from miles.backends.training_utils.weight_update.hf_weight_iterator import WeightUpdatePlacement - - -class _Iterator(MegatronHfWeightIteratorBase): - def _iter_hf_param_units(self, weights, *, materialize): - return iter(()) - - def _export_pp_local_lora(self, adapter): - return [] - - -def _make(speculative, mtp_num_layers): - model = [SimpleNamespace(config=SimpleNamespace(mtp_num_layers=mtp_num_layers))] - args = Namespace(sglang_speculative_algorithm=speculative, q_lora_rank=None) - return _Iterator( - args, model, placement=WeightUpdatePlacement(gather_pp=True), model_name="qwen", quantization_config=None - ) - - -@pytest.mark.parametrize( - "speculative, mtp_num_layers, selector", - [ - (None, None, "all"), - (None, 1, "all"), - ("EAGLE", 1, "all"), - ("EAGLE", None, "target"), - ("EAGLE", 0, "target"), - ], - ids=["no_draft", "no_draft_with_mtp", "trained_draft", "frozen_draft", "frozen_draft_zero_layers"], -) -def test_frozen_draft_is_excluded_from_the_sync(speculative, mtp_num_layers, selector): - """A draft the trainer never feeds must not be re-postprocessed on every sync.""" - assert _make(speculative, mtp_num_layers).weight_update_selector == selector diff --git a/tests/fast/backends/training_utils/weight_update/test_dist_weight_update_lifecycle.py b/tests/fast/backends/training_utils/weight_update/test_dist_weight_update_lifecycle.py index 15c16ace0df..dc2714ec2e3 100644 --- a/tests/fast/backends/training_utils/weight_update/test_dist_weight_update_lifecycle.py +++ b/tests/fast/backends/training_utils/weight_update/test_dist_weight_update_lifecycle.py @@ -88,8 +88,12 @@ def _make_updater(engines: list[_RecordingApiClient], *, pause_generation_mode: ) iterator = MagicMock() iterator.iter_hf_weights.return_value = iter([]) - iterator.weight_update_selector = "all" - args = Namespace(pause_generation_mode=pause_generation_mode, check_lora_weight_equal=False) + args = Namespace( + pause_generation_mode=pause_generation_mode, + check_lora_weight_equal=False, + sglang_speculative_algorithm=None, + enable_mtp_training=False, + ) with patch(f"{_UPDATER_MODULE}.get_weight_transfer_protocol", return_value=protocol): return WeightUpdater( args, diff --git a/tests/fast/backends/training_utils/weight_update/test_lora_weight_sync_validation.py b/tests/fast/backends/training_utils/weight_update/test_lora_weight_sync_validation.py index a9c2b819fa7..3c138a8f6e7 100644 --- a/tests/fast/backends/training_utils/weight_update/test_lora_weight_sync_validation.py +++ b/tests/fast/backends/training_utils/weight_update/test_lora_weight_sync_validation.py @@ -57,6 +57,8 @@ def _make_args(**overrides): actor_num_nodes=1, actor_num_gpus_per_node=1, pause_generation_mode="retract", + sglang_speculative_algorithm=None, + enable_mtp_training=False, ) defaults.update(overrides) return Namespace(**defaults) diff --git a/tests/fast/backends/training_utils/weight_update/test_weight_update_selector.py b/tests/fast/backends/training_utils/weight_update/test_weight_update_selector.py new file mode 100644 index 00000000000..9ac39e0992a --- /dev/null +++ b/tests/fast/backends/training_utils/weight_update/test_weight_update_selector.py @@ -0,0 +1,28 @@ +"""The weight-update selector follows whether the trainer trains MTP layers, on every backend.""" + +from tests.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=60, suite="stage-b-cpu", labels=["weight-update"]) + +from argparse import Namespace + +import pytest + +from miles.backends.training_utils.weight_update.session import weight_update_selector + + +@pytest.mark.parametrize( + "speculative, enable_mtp_training, selector", + [ + (None, False, "all"), + (None, True, "all"), + ("EAGLE", True, "all"), + ("EAGLE", False, "target"), + ("EAGLE3", False, "target"), + ], + ids=["no_draft", "no_draft_with_mtp", "trained_draft", "frozen_draft", "external_draft"], +) +def test_a_draft_is_synced_only_when_the_trainer_trains_mtp_layers(speculative, enable_mtp_training, selector): + """FSDP and torchtitan reject --enable-mtp-training, so their drafts are always left out.""" + args = Namespace(sglang_speculative_algorithm=speculative, enable_mtp_training=enable_mtp_training) + assert weight_update_selector(args) == selector diff --git a/tests/fast/ray/rollout/conftest.py b/tests/fast/ray/rollout/conftest.py index cd012d79f37..10c1099939e 100644 --- a/tests/fast/ray/rollout/conftest.py +++ b/tests/fast/ray/rollout/conftest.py @@ -46,6 +46,7 @@ def make_args(**overrides: Any) -> Namespace: update_weights_interval=1, check_weight_update_equal=False, check_weight_update_skip_list=None, + check_weight_update_selector="all", # batch / training global_batch_size=8, use_dynamic_global_batch_size=False, diff --git a/tests/fast/ray/rollout/test_server_cell_state_machine.py b/tests/fast/ray/rollout/test_server_cell_state_machine.py index 060b70612e6..19d865835a0 100644 --- a/tests/fast/ray/rollout/test_server_cell_state_machine.py +++ b/tests/fast/ray/rollout/test_server_cell_state_machine.py @@ -554,6 +554,19 @@ async def test_a_cell_primes_the_weight_update_checker_before_its_first_update(s assert [kwargs["action"] for kwargs in checker_calls] == ["snapshot", "reset_tensors"] assert checker_calls[1]["skip_list"] == ["a"] + async def test_the_checker_snapshots_and_resets_only_the_runners_it_compares(self, cell_env): + """A draft the sync leaves out would stay scrambled for the rest of the run.""" + cell = _make_cell(args_overrides=dict(check_weight_update_equal=True, check_weight_update_selector="target")) + await cell.init() + + await cell.tick() + + checker_calls = [kwargs for name, kwargs in cell_env["memory_calls"] if name == "check_weights"] + assert [(kwargs["action"], kwargs["selector"]) for kwargs in checker_calls] == [ + ("snapshot", "target"), + ("reset_tensors", "target"), + ] + async def test_the_weight_update_checker_stays_out_of_the_way_when_it_is_disabled(self, cell_env): """The checker is a ci-only tool and must not touch engines in a normal run.""" cell = _make_cell() diff --git a/tests/fast/utils/test_arguments.py b/tests/fast/utils/test_arguments.py index f9c179244f2..05fbe7b8ba3 100644 --- a/tests/fast/utils/test_arguments.py +++ b/tests/fast/utils/test_arguments.py @@ -1701,6 +1701,10 @@ def test_a_run_that_does_not_train_mtp_builds_no_mtp_layer(self, backend): """Megatron trains every MTP layer it builds, so the model script's layer count alone builds none.""" assert self._validate(["--train-backend", backend, "--mtp-num-layers", "1"]).mtp_num_layers is None + def test_an_explicit_zero_becomes_megatrons_no_mtp_value(self): + """Megatron treats any non-None mtp_num_layers as an MTP model, so 0 must not reach it.""" + assert self._validate(["--mtp-num-layers", "0"]).mtp_num_layers is None + def test_mtp_training_keeps_the_layers(self): assert self._validate(["--enable-mtp-training", "--mtp-num-layers", "1"]).mtp_num_layers == 1 @@ -3078,3 +3082,28 @@ def test_fully_async_without_over_sampling_batch_size_does_not_warn(caplog): _resolve_rollout_functions(args) assert "over-sampling-batch-size" not in caplog.text + + +class TestCheckWeightUpdateSelectorDefault: + """reset_tensors scrambles every tensor the checker covers, so by default it covers what the sync rewrites.""" + + def _validate(self, extra): + parser = argparse.ArgumentParser() + get_miles_extra_args_provider()(parser) + args = parser.parse_args(extra + ["--num-rollout", "1"] + REQUIRED_ARGS) + _set_megatron_parallel_sizes(args) + miles_validate_args(args) + return args + + @pytest.mark.parametrize( + ("extra", "selector"), + [ + ([], "all"), + (["--sglang-speculative-algorithm", "EAGLE"], "target"), + (["--sglang-speculative-algorithm", "EAGLE", "--enable-mtp-training", "--mtp-num-layers", "1"], "all"), + (["--sglang-speculative-algorithm", "EAGLE", "--check-weight-update-selector", "draft"], "draft"), + ], + ids=["no_draft", "frozen_draft", "trained_draft", "explicit"], + ) + def test_the_checker_covers_what_the_sync_rewrites_unless_narrowed(self, extra, selector): + assert self._validate(extra).check_weight_update_selector == selector