diff --git a/docker/patch/latest/sglang.patch b/docker/patch/latest/sglang.patch index 0cf9731bb0..8cea544bd1 100644 --- a/docker/patch/latest/sglang.patch +++ b/docker/patch/latest/sglang.patch @@ -719,7 +719,7 @@ index ca54a931b..6c102a251 100644 if enable_dual_stream: current_stream = torch.cuda.current_stream() diff --git a/python/sglang/srt/layers/attention/nsa/utils.py b/python/sglang/srt/layers/attention/nsa/utils.py -index 00ef96f9b..9885fcd0d 100644 +index 00ef96f9b..5adaec804 100644 --- a/python/sglang/srt/layers/attention/nsa/utils.py +++ b/python/sglang/srt/layers/attention/nsa/utils.py @@ -54,7 +54,12 @@ def can_nsa_prefill_cp_round_robin_split(forward_batch: "ForwardBatch"): @@ -775,20 +775,32 @@ index 00ef96f9b..9885fcd0d 100644 if can_nsa_prefill_cp_round_robin_split(forward_batch): tokens = ceil_div(tokens, attn_cp_size) return tokens -@@ -152,10 +166,9 @@ class NSAContextParallelMetadata: +@@ -152,10 +166,20 @@ class NSAContextParallelMetadata: def can_cp_split(seq_len: int, cp_size: int, use_nsa: bool, forward_batch): if is_nsa_prefill_cp_round_robin_split(): -+ if seq_len < cp_size or seq_len % cp_size != 0: -+ return False - cur_cp_seq_len = seq_len // cp_size +- cur_cp_seq_len = seq_len // cp_size - assert ( - seq_len % cp_size == 0 - ), f"seq_len {seq_len} is not divisible by cp_size {cp_size} when nsa_prefill_cp_mode is round-robin-split" ++ # Use actual extend sequence length instead of (possibly padded) input_ids ++ # length to stay consistent with can_nsa_prefill_cp_round_robin_split(), ++ # which also checks sum(extend_seq_lens_cpu). When prepare_mlp_sync_batch ++ # pads input_ids to ceil_align(n, attn_cp_size), len(input_ids) can become ++ # divisible by cp_size even though the real extend length is not, causing ++ # hidden_states to be CP-split while the attention metadata is not. ++ actual_seq_len = ( ++ sum(forward_batch.extend_seq_lens_cpu) ++ if forward_batch.extend_seq_lens_cpu is not None ++ else seq_len ++ ) ++ if actual_seq_len < cp_size or actual_seq_len % cp_size != 0: ++ return False ++ cur_cp_seq_len = actual_seq_len // cp_size else: # TODO current just support prefill batch=1 and len(input_ids) > self.cp_size * 2 # Note: (self.cp_size * 2) To achieve load balancing for seq computation, -@@ -175,10 +188,6 @@ def can_cp_split(seq_len: int, cp_size: int, use_nsa: bool, forward_batch): +@@ -175,10 +199,6 @@ def can_cp_split(seq_len: int, cp_size: int, use_nsa: bool, forward_batch): def cp_split_and_rebuild_data(forward_batch, input_: torch.Tensor): if is_nsa_prefill_cp_round_robin_split(): @@ -799,7 +811,7 @@ index 00ef96f9b..9885fcd0d 100644 return nsa_cp_round_robin_split_data(input_) input_list = list( -@@ -192,11 +201,6 @@ def cp_split_and_rebuild_data(forward_batch, input_: torch.Tensor): +@@ -192,11 +212,6 @@ def cp_split_and_rebuild_data(forward_batch, input_: torch.Tensor): def cp_split_and_rebuild_position(forward_batch, positions: torch.Tensor): if is_nsa_prefill_cp_round_robin_split(): @@ -2402,6 +2414,122 @@ index cc673a9ca..06c430d2c 100644 if hasattr(backend, "use_mha") and backend.use_mha: return AttnForwardMethod.MHA_ONE_SHOT return AttnForwardMethod.MLA +diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py +index cb13a7c67..d62111471 100644 +--- a/python/sglang/srt/models/deepseek_nextn.py ++++ b/python/sglang/srt/models/deepseek_nextn.py +@@ -29,6 +29,7 @@ from sglang.srt.layers.attention.nsa.utils import ( + can_cp_split, + cp_all_gather_rerange_output, + cp_split_and_rebuild_data, ++ cp_split_and_rebuild_position, + is_nsa_enable_prefill_cp, + nsa_use_prefill_cp, + prepare_input_dp_with_cp_dsa, +@@ -160,6 +161,7 @@ class DeepseekModelNextN(nn.Module): + + if nsa_use_prefill_cp(forward_batch, self.nsa_enable_prefill_cp): + hidden_states = cp_split_and_rebuild_data(forward_batch, hidden_states) ++ positions = cp_split_and_rebuild_position(forward_batch, positions) + residual = None + with get_global_expert_distribution_recorder().disable_this_region(): + hidden_states, residual = self.decoder( +diff --git a/python/sglang/srt/models/glm4v_moe.py b/python/sglang/srt/models/glm4v_moe.py +index 324de18b4..c99723f49 100644 +--- a/python/sglang/srt/models/glm4v_moe.py ++++ b/python/sglang/srt/models/glm4v_moe.py +@@ -52,11 +52,31 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): + self.num_fused_shared_experts = 0 + self.determine_num_fused_shared_experts() + +- self.model = Glm4MoeModel( +- config, +- quant_config, +- prefix=add_prefix("language_model", prefix), +- ) ++ if not self.config.encoder_only: ++ self.model = Glm4MoeModel( ++ config, ++ quant_config, ++ prefix=add_prefix("language_model", prefix), ++ ) ++ ++ if self.pp_group.is_last_rank: ++ if self.pp_group.world_size == 1 and self.config.tie_word_embeddings: ++ self.lm_head = self.model.embed_tokens ++ else: ++ self.lm_head = ParallelLMHead( ++ config.vocab_size, ++ config.hidden_size, ++ quant_config=quant_config, ++ prefix=add_prefix("lm_head", prefix), ++ use_attn_tp_group=get_global_server_args().enable_dp_lm_head, ++ ) ++ else: ++ # ranks other than the last rank will have a placeholder layer ++ self.lm_head = PPMissingLayer() ++ else: ++ # encoder_only mode: no language model, so no lm_head needed ++ self.lm_head = None ++ + self.visual = Glm4vVisionModel( + config.vision_config, + quant_config=quant_config, +@@ -64,21 +84,6 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): + use_data_parallel=self.use_data_parallel, + ) + +- if self.pp_group.is_last_rank: +- if self.pp_group.world_size == 1 and self.config.tie_word_embeddings: +- self.lm_head = self.model.embed_tokens +- else: +- self.lm_head = ParallelLMHead( +- config.vocab_size, +- config.hidden_size, +- quant_config=quant_config, +- prefix=add_prefix("lm_head", prefix), +- use_attn_tp_group=get_global_server_args().enable_dp_lm_head, +- ) +- else: +- # ranks other than the last rank will have a placeholder layer +- self.lm_head = PPMissingLayer() +- + self.logits_processor = LogitsProcessor(config) + self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) + self.is_mrope_enabled = "mrope_section" in self.config.rope_scaling +@@ -219,6 +224,11 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue ++ # Skip loading visual/language model weights ++ if ( ++ self.config.encoder_only or self.config.language_only ++ ) and name not in params_dict: ++ continue + if name not in params_dict: + continue + +@@ -234,6 +244,8 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue ++ if "visual" in name or self.config.encoder_only: ++ continue + + # Mark as expert weight regardless of whether we can process it + is_expert_weight = True +@@ -265,6 +277,11 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue ++ # Skip loading mm/language parameters ++ if ( ++ self.config.encoder_only or self.config.language_only ++ ) and name not in params_dict: ++ continue + if name not in params_dict: + continue + diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 2cf813bce..1250c49e4 100644 --- a/python/sglang/srt/models/gpt_oss.py @@ -2517,7 +2645,7 @@ index 2cf813bce..1250c49e4 100644 weights_out_dict = dict(weights_in) diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py -index f012254..1dad8bb 100644 +index f01225487..1dad8bb8e 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -372,6 +372,7 @@ class Qwen3_5LinearDecoderLayer(nn.Module): @@ -2620,6 +2748,66 @@ index d641826e3..3abc39ef3 100644 hidden_states, residual = layer( positions, hidden_states, +diff --git a/python/sglang/srt/multimodal/processors/glm4v.py b/python/sglang/srt/multimodal/processors/glm4v.py +index 33cce6fe2..0970c4550 100644 +--- a/python/sglang/srt/multimodal/processors/glm4v.py ++++ b/python/sglang/srt/multimodal/processors/glm4v.py +@@ -1,6 +1,9 @@ + from typing import List, Union + ++import torch ++ + from sglang.srt.layers.rotary_embedding import MRotaryEmbedding ++from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem + from sglang.srt.models.glm4v import Glm4vForConditionalGeneration + from sglang.srt.models.glm4v_moe import Glm4vMoeForConditionalGeneration + from sglang.srt.multimodal.processors.base_processor import ( +@@ -45,6 +48,8 @@ class Glm4vImageProcessor(SGLangBaseProcessor): + self.IMAGE_END_TOKEN_ID = hf_config.image_end_token_id + self.VIDEO_START_TOKEN_ID = hf_config.video_start_token_id + self.VIDEO_END_TOKEN_ID = hf_config.video_end_token_id ++ self.IM_START_TOKEN_ID = self.IMAGE_START_TOKEN_ID ++ self.IM_END_TOKEN_ID = self.IMAGE_END_TOKEN_ID + + # Vision config + self.IMAGE_FACTOR = 28 +@@ -59,6 +64,36 @@ class Glm4vImageProcessor(SGLangBaseProcessor): + video_token_id=self.IM_TOKEN_ID, + ).build(_processor) + ++ def get_mm_data(self, prompt, embeddings, img_grid_thw): ++ input_ids, offsets = self.build_input_ids(prompt, img_grid_thw) ++ mm_items = [ ++ MultimodalDataItem( ++ modality=Modality.IMAGE, ++ offsets=offsets, ++ precomputed_embeddings=embeddings, ++ ) ++ ] ++ ++ input_ids_tensor = torch.tensor(input_ids) ++ mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index_glm4v( ++ input_ids=input_ids_tensor.unsqueeze(0), ++ hf_config=self.hf_config, ++ image_grid_thw=img_grid_thw, ++ video_grid_thw=None, ++ attention_mask=None, ++ ) ++ mrope_positions = mrope_positions.squeeze(1) ++ ++ return { ++ "input_ids": input_ids, ++ "mm_items": mm_items, ++ "im_start_id": self.IM_START_TOKEN_ID, ++ "im_end_id": self.IM_END_TOKEN_ID, ++ "im_token_id": self.IM_TOKEN_ID, ++ "mrope_positions": mrope_positions, ++ "mrope_position_delta": mrope_position_delta, ++ } ++ + async def process_mm_data_async( + self, + image_data: List[Union[str, bytes]], diff --git a/python/sglang/srt/multimodal/processors/qwen_vl.py b/python/sglang/srt/multimodal/processors/qwen_vl.py index 4395654e4..f9b5ea4ab 100644 --- a/python/sglang/srt/multimodal/processors/qwen_vl.py diff --git a/docker/version.txt b/docker/version.txt index 88b5642b1b..993e2548ed 100644 --- a/docker/version.txt +++ b/docker/version.txt @@ -1 +1 @@ -nightly-dev-20260309a +nightly-dev-20260311a diff --git a/slime/ray/rollout.py b/slime/ray/rollout.py index 3df73c41da..7ea0d12b56 100644 --- a/slime/ray/rollout.py +++ b/slime/ray/rollout.py @@ -1007,7 +1007,7 @@ def start_rollout_servers(args, pg) -> dict[str, RolloutServer]: has_epd = model_cfg.has_encoder_disaggregation - def _make_group(group_cfg, overrides_extra=None): + def _make_group(group_cfg, router_ip, router_port, overrides_extra=None): nonlocal engine_offset, gpu_offset gpus_per_engine = group_cfg.num_gpus_per_engine num_gpu_per_engine_local = min(gpus_per_engine, args.num_gpus_per_node) @@ -1051,7 +1051,7 @@ def _make_group(group_cfg, overrides_extra=None): for group_cfg in model_cfg.server_groups: if group_cfg.worker_type != "encoder": continue - group = _make_group(group_cfg) + group = _make_group(group_cfg, router_ip, router_port) handles, port_cursors = group.start_engines(port_cursors) if handles: ray.get(handles) @@ -1070,7 +1070,7 @@ def _make_group(group_cfg, overrides_extra=None): if encoder_urls and group_cfg.worker_type == "prefill": overrides_extra["language_only"] = True overrides_extra["encoder_urls"] = encoder_urls - group = _make_group(group_cfg, overrides_extra=overrides_extra) + group = _make_group(group_cfg, router_ip, router_port, overrides_extra=overrides_extra) handles, port_cursors = group.start_engines(port_cursors) non_encoder_handles.extend(handles) server_groups.append(group) @@ -1081,7 +1081,7 @@ def _make_group(group_cfg, overrides_extra=None): # No EPD — start all groups in one pass (original path). all_init_handles: list = [] for group_cfg in model_cfg.server_groups: - group = _make_group(group_cfg) + group = _make_group(group_cfg, router_ip, router_port) handles, port_cursors = group.start_engines(port_cursors) all_init_handles.extend(handles) server_groups.append(group) diff --git a/slime_plugins/mbridge/qwen3_5.py b/slime_plugins/mbridge/qwen3_5.py index 2447181102..1f2ee73eac 100644 --- a/slime_plugins/mbridge/qwen3_5.py +++ b/slime_plugins/mbridge/qwen3_5.py @@ -180,7 +180,9 @@ def _weight_name_mapping_mtp_mlp(self, name: str) -> list[str]: if keyword in name: if "{expert_id}" in mapping_names[0]: expert_id = name.split("weight")[-1] - convert_names.extend([x.format(layer_number=layer_number, expert_id=expert_id) for x in mapping_names]) + convert_names.extend( + [x.format(layer_number=layer_number, expert_id=expert_id) for x in mapping_names] + ) else: convert_names.extend([x.format(layer_number=layer_number) for x in mapping_names]) break diff --git a/tests/test_qwen2.5_0.5B_async_short.py b/tests/test_qwen2.5_0.5B_async_short.py new file mode 100644 index 0000000000..43a8f0a156 --- /dev/null +++ b/tests/test_qwen2.5_0.5B_async_short.py @@ -0,0 +1,121 @@ +import os +import slime.utils.external_utils.command_utils as U + +TIGHT_DEVICE_MEMORY = U.get_bool_env_var("SLIME_TEST_TIGHT_DEVICE_MEMORY", "1") + +MODEL_NAME = "Qwen2.5-0.5B-Instruct" +MODEL_TYPE = "qwen2.5-0.5B" +NUM_GPUS = 4 + + +def prepare(): + U.exec_command("mkdir -p /root/models /root/datasets") + U.exec_command(f"huggingface-cli download Qwen/{MODEL_NAME} --local-dir /root/models/{MODEL_NAME}") + U.hf_download_dataset("zhuzilin/dapo-math-17k") + + +def execute(): + ckpt_args = f"--hf-checkpoint /root/models/{MODEL_NAME}/ " f"--ref-load /root/models/{MODEL_NAME}/ " + + rollout_args = ( + "--prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl " + "--input-key prompt " + "--label-key label " + "--apply-chat-template " + "--rollout-shuffle " + "--rm-type deepscaler " + "--num-rollout 3 " + "--rollout-batch-size 8 " + "--n-samples-per-prompt 4 " + "--rollout-max-response-len 8192 " + "--rollout-temperature 0.8 " + "--global-batch-size 32 " + "--balance-data " + ) + + perf_args = ( + "--tensor-model-parallel-size 1 " + "--sequence-parallel " + "--pipeline-model-parallel-size 1 " + "--context-parallel-size 1 " + "--expert-model-parallel-size 1 " + "--expert-tensor-parallel-size 1 " + "--use-dynamic-batch-size " + "--max-tokens-per-gpu 9216 " + ) + + grpo_args = ( + "--advantage-estimator grpo " + "--use-kl-loss " + "--kl-loss-coef 0.00 " + "--kl-loss-type low_var_kl " + "--entropy-coef 0.00 " + "--eps-clip 0.2 " + "--eps-clip-high 0.28 " + ) + + optimizer_args = ( + "--optimizer adam " + "--lr 1e-6 " + "--lr-decay-style constant " + "--weight-decay 0.1 " + "--adam-beta1 0.9 " + "--adam-beta2 0.98 " + ) + + sglang_args = ( + "--rollout-num-gpus-per-engine 1 " + f"--sglang-mem-fraction-static {0.55 if TIGHT_DEVICE_MEMORY else 0.65} " + "--sglang-cuda-graph-max-bs 32 " + "--sglang-enable-metrics " + ) + + ci_args = "--ci-test " + + fault_tolerance_args = ( + "--use-fault-tolerance " + "--rollout-health-check-interval 5 " + "--rollout-health-check-timeout 10 " + "--rollout-health-check-first-wait 0 " + ) + + misc_args = ( + "--attention-dropout 0.0 " + "--hidden-dropout 0.0 " + "--accumulate-allreduce-grads-in-fp32 " + "--attention-softmax-in-fp32 " + "--attention-backend flash " + "--actor-num-nodes 1 " + "--actor-num-gpus-per-node 1 " + "--rollout-num-gpus 3 " + "--megatron-to-hf-mode bridge " + ) + + train_args = ( + f"{ckpt_args} " + f"{rollout_args} " + f"{optimizer_args} " + f"{grpo_args} " + f"{U.get_default_wandb_args(__file__)} " + f"{perf_args} " + f"{sglang_args} " + f"{ci_args} " + f"{fault_tolerance_args} " + f"{misc_args} " + ) + + U.execute_train( + train_args=train_args, + num_gpus_per_node=NUM_GPUS, + megatron_model_type=MODEL_TYPE, + train_script="train_async.py", + ) + + +if __name__ == "__main__": + prepare() + os.environ.pop("http_proxy") + os.environ.pop("https_proxy") + os.environ.pop("HTTP_PROXY") + os.environ.pop("HTTPS_PROXY") + execute() diff --git a/tests/test_qwen2.5_0.5B_short.py b/tests/test_qwen2.5_0.5B_short.py new file mode 100644 index 0000000000..c5b3157848 --- /dev/null +++ b/tests/test_qwen2.5_0.5B_short.py @@ -0,0 +1,120 @@ +import os +import slime.utils.external_utils.command_utils as U + +TIGHT_DEVICE_MEMORY = U.get_bool_env_var("SLIME_TEST_TIGHT_DEVICE_MEMORY", "1") + +MODEL_NAME = "Qwen2.5-0.5B-Instruct" +MODEL_TYPE = "qwen2.5-0.5B" +NUM_GPUS = 4 + + +def prepare(): + U.exec_command("mkdir -p /root/models /root/datasets") + U.exec_command(f"huggingface-cli download Qwen/{MODEL_NAME} --local-dir /root/models/{MODEL_NAME}") + U.hf_download_dataset("zhuzilin/dapo-math-17k") + + +def execute(): + ckpt_args = f"--hf-checkpoint /root/models/{MODEL_NAME}/ " f"--ref-load /root/models/{MODEL_NAME}/ " + + rollout_args = ( + "--prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl " + "--input-key prompt " + "--label-key label " + "--apply-chat-template " + "--rollout-shuffle " + "--rm-type deepscaler " + "--num-rollout 3 " + "--rollout-batch-size 8 " + "--n-samples-per-prompt 4 " + "--rollout-max-response-len 8192 " + "--rollout-temperature 0.8 " + "--global-batch-size 32 " + "--balance-data " + ) + + perf_args = ( + "--tensor-model-parallel-size 1 " + "--sequence-parallel " + "--pipeline-model-parallel-size 1 " + "--context-parallel-size 1 " + "--expert-model-parallel-size 1 " + "--expert-tensor-parallel-size 1 " + "--use-dynamic-batch-size " + "--max-tokens-per-gpu 9216 " + ) + + grpo_args = ( + "--advantage-estimator grpo " + "--use-kl-loss " + "--kl-loss-coef 0.00 " + "--kl-loss-type low_var_kl " + "--entropy-coef 0.00 " + "--eps-clip 0.2 " + "--eps-clip-high 0.28 " + ) + + optimizer_args = ( + "--optimizer adam " + "--lr 1e-6 " + "--lr-decay-style constant " + "--weight-decay 0.1 " + "--adam-beta1 0.9 " + "--adam-beta2 0.98 " + ) + + sglang_args = ( + "--rollout-num-gpus-per-engine 1 " + f"--sglang-mem-fraction-static {0.6 if TIGHT_DEVICE_MEMORY else 0.7} " + "--sglang-cuda-graph-max-bs 32 " + "--sglang-enable-metrics " + ) + + ci_args = "--ci-test " + + fault_tolerance_args = ( + "--use-fault-tolerance " + "--rollout-health-check-interval 5 " + "--rollout-health-check-timeout 10 " + "--rollout-health-check-first-wait 0 " + ) + + misc_args = ( + "--attention-dropout 0.0 " + "--hidden-dropout 0.0 " + "--accumulate-allreduce-grads-in-fp32 " + "--attention-softmax-in-fp32 " + "--attention-backend flash " + "--actor-num-nodes 1 " + "--actor-num-gpus-per-node 4 " + "--colocate " + "--megatron-to-hf-mode bridge " + ) + + train_args = ( + f"{ckpt_args} " + f"{rollout_args} " + f"{optimizer_args} " + f"{grpo_args} " + f"{U.get_default_wandb_args(__file__)} " + f"{perf_args} " + f"{sglang_args} " + f"{ci_args} " + f"{fault_tolerance_args} " + f"{misc_args} " + ) + + U.execute_train( + train_args=train_args, + num_gpus_per_node=NUM_GPUS, + megatron_model_type=MODEL_TYPE, + ) + + +if __name__ == "__main__": + prepare() + os.environ.pop("http_proxy") + os.environ.pop("https_proxy") + os.environ.pop("HTTP_PROXY") + os.environ.pop("HTTPS_PROXY") + execute() diff --git a/tests/test_qwen3_5_mtp_bridge_mapping.py b/tests/test_qwen3_5_mtp_bridge_mapping.py index b94753f41d..c8a970ef29 100644 --- a/tests/test_qwen3_5_mtp_bridge_mapping.py +++ b/tests/test_qwen3_5_mtp_bridge_mapping.py @@ -13,8 +13,9 @@ def install_bridge_stubs(): models_mod = types.ModuleType("megatron.core.models") gpt_mod = types.ModuleType("megatron.core.models.gpt") gpt_layer_specs_mod = types.ModuleType("megatron.core.models.gpt.gpt_layer_specs") - gpt_layer_specs_mod.get_gpt_mtp_block_spec = ( - lambda _config, transformer_layer_spec, **_kwargs: ("mtp-spec", transformer_layer_spec) + gpt_layer_specs_mod.get_gpt_mtp_block_spec = lambda _config, transformer_layer_spec, **_kwargs: ( + "mtp-spec", + transformer_layer_spec, ) mbridge_mod = types.ModuleType("mbridge") @@ -114,7 +115,9 @@ def load_bridge_module(): def load_raw_export_module(): - module_path = Path(__file__).resolve().parents[1] / "slime" / "backends" / "megatron_utils" / "megatron_to_hf" / "qwen3_5.py" + module_path = ( + Path(__file__).resolve().parents[1] / "slime" / "backends" / "megatron_utils" / "megatron_to_hf" / "qwen3_5.py" + ) module_name = "test_qwen3_5_raw_export_module" sys.modules.pop(module_name, None) spec = importlib.util.spec_from_file_location(module_name, module_path)