diff --git a/src/megatron/bridge/data/energon/energon_provider.py b/src/megatron/bridge/data/energon/energon_provider.py index f33ea48dc1..ded5f8d308 100644 --- a/src/megatron/bridge/data/energon/energon_provider.py +++ b/src/megatron/bridge/data/energon/energon_provider.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os from dataclasses import dataclass from typing import Any, Optional @@ -33,13 +34,38 @@ class EnergonProvider(DatasetProvider): num_workers: int_repr dataloader_type: str = "external" task_encoder: Optional[Any] = None - # Enable batch-level online sequence packing + # Existing in-batch packing switch. + # Semantics: pack samples *within each already formed micro-batch*. + # This path keeps historical behavior used by existing recipes. pack_sequences_in_batch: bool = False + # THD dataloader switch for batch-level online packing. + # Semantics: pack samples *across a dataloader-side candidate buffer* + # before a micro-batch is formed. + # This mode is independent from pack_sequences_in_batch. + batch_level_packing: bool = False + packing_buffer_size: Optional[int] = None + shuffle_buffer_size: int = 100 + # Optional bin selector for datasets split into bin directories. + # Used to pin data selection and align comparisons with Energon BSHD. + cord_bins_root: Optional[str] = None + cord_bin_prefix: str = "cord_bin_" + cord_bin_id: Optional[str] = None def build_datasets(self, context: DatasetBuildContext): - assert self.path, "EnergonProvider.path must be set. Use CLI override: dataset.path=" + resolved_path = self.path + if self.cord_bin_id is not None and self.cord_bin_id != "": + assert self.cord_bins_root, ( + "EnergonProvider.cord_bins_root must be set when dataset.cord_bin_id is provided." + ) + resolved_path = os.path.join(self.cord_bins_root, f"{self.cord_bin_prefix}{self.cord_bin_id}") + + assert resolved_path, "EnergonProvider.path must be set. Use CLI override: dataset.path=" + if self.task_encoder is not None and hasattr(self.task_encoder, "seq_len"): + self.task_encoder.seq_len = self.seq_length + self.task_encoder.seq_length = self.seq_length + effective_packing_buffer_size = self.packing_buffer_size if self.batch_level_packing else None dataset = EnergonMultiModalDataModule( - path=self.path, + path=resolved_path, tokenizer=context.tokenizer if context.tokenizer is not None else self.tokenizer, image_processor=self.image_processor, seq_length=self.seq_length, @@ -47,6 +73,8 @@ def build_datasets(self, context: DatasetBuildContext): micro_batch_size=self.micro_batch_size, global_batch_size=self.global_batch_size, num_workers=self.num_workers, + packing_buffer_size=effective_packing_buffer_size, + shuffle_buffer_size=self.shuffle_buffer_size, pg_collection=context.pg_collection, ) return ( diff --git a/src/megatron/bridge/models/conversion/auto_bridge.py b/src/megatron/bridge/models/conversion/auto_bridge.py index 0c61b75d47..ddb411142c 100644 --- a/src/megatron/bridge/models/conversion/auto_bridge.py +++ b/src/megatron/bridge/models/conversion/auto_bridge.py @@ -30,7 +30,6 @@ from megatron.core.transformer.module import MegatronModule from megatron.core.transformer.transformer_config import MLATransformerConfig, TransformerConfig -from modelopt.torch.quantization.utils import is_quantized from safetensors.torch import save_file from transformers.configuration_utils import PretrainedConfig from typing_extensions import Unpack @@ -51,6 +50,21 @@ logger = logging.getLogger(__name__) +try: + from modelopt.torch.quantization.utils import is_quantized +except ImportError as exc: + # modelopt/scipy is optional for recipe import and training startup. + # Keep quantization-only export path available when dependency exists. + logger.warning( + "modelopt quantization utils unavailable; quantized-export detection disabled: %s", + exc, + ) + + def is_quantized(_model: object) -> bool: + """Fallback quantization probe when ModelOpt is unavailable.""" + return False + + MegatronModelT = TypeVar("MegatronModelT", bound=MegatronModule) DataclassT = TypeVar("DataclassT") diff --git a/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/model.py b/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/model.py index 81b9e853b1..b753b05e51 100644 --- a/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/model.py +++ b/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/model.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging from typing import Optional import torch @@ -42,16 +43,23 @@ collapse_thw, get_dist_train_vision_dp_data, get_vision_cp_data, + is_rank_0, pack_dist_train_vision_module_output, preprocess_packed_seqs, qwen3vl_cp_split, reorganize_inputs, split_data_cp_rank, split_deepstack_embs, + thd_diag_align_enabled, + thd_diag_enabled, + thd_diag_mrope_enabled, ) from megatron.bridge.models.qwen_vl.modelling_qwen3_vl.vision_model import Qwen3VLVisionModel +logger = logging.getLogger(__name__) + + class Qwen3VLModel(MegatronModule): """Qwen3VL multi-modal model. @@ -349,6 +357,8 @@ def forward( video_input_mask: torch.Tensor = None, cp_img_num: list[int] = None, images_padded: list[bool] = None, + rope_cu_seqlens: torch.Tensor = None, + moe_padding_mask: torch.Tensor = None, inference_context: object | None = None, runtime_gather_output: bool | None = None, mm_token_type_ids: torch.Tensor = None, @@ -399,6 +409,19 @@ def forward( # so it must be a real tensor. For packed sequences we use the THD-format # input_ids_thd (updated below); for regular sequences we use input_ids as-is. lm_input_ids = input_ids + moe_padding_mask_for_lm = moe_padding_mask + + def _mask_summary(mask: torch.Tensor, max_items: int = 8) -> tuple[int, str]: + flat_idx = torch.nonzero(mask.reshape(-1), as_tuple=False).view(-1) + count = int(flat_idx.numel()) + if count == 0: + return count, "[]" + head = flat_idx[:max_items].tolist() + tail = flat_idx[-max_items:].tolist() if count > max_items else [] + summary = f"head={head}" + if tail: + summary += f", tail={tail}" + return count, summary if self.pre_process: # can reorganize_inputs at dataset @@ -506,10 +529,60 @@ def forward( if packed_seq_params is not None: if attention_mask is None: attention_mask = torch.ones_like(input_ids, dtype=torch.bool, device=input_ids.device) + attn_mask_bool = attention_mask.bool() input_ids_thd, _ = preprocess_packed_seqs( - input_ids, attention_mask, pre_process=True, pg_collection=self.pg_collection + input_ids, attn_mask_bool, pre_process=True, pg_collection=self.pg_collection ) lm_input_ids = input_ids_thd + if moe_padding_mask_for_lm is not None: + moe_padding_mask_for_lm = preprocess_packed_seqs( + moe_padding_mask_for_lm.to(dtype=torch.int32), + attn_mask_bool, + pre_process=True, + pg_collection=self.pg_collection, + )[0].bool() + if thd_diag_align_enabled() and is_rank_0() and labels is not None and loss_mask is not None: + labels_thd = preprocess_packed_seqs( + labels, + attn_mask_bool, + pre_process=True, + pg_collection=self.pg_collection, + )[0] + loss_mask_thd = preprocess_packed_seqs( + loss_mask.to(dtype=torch.float32), + attn_mask_bool, + pre_process=True, + pg_collection=self.pg_collection, + )[0] + labels_valid_pre = labels.ne(-100) + labels_valid_post = labels_thd.ne(-100) + loss_valid_pre = loss_mask > 0 + loss_valid_post = loss_mask_thd > 0 + label_pre_cnt, label_pre_idx = _mask_summary(labels_valid_pre) + label_post_cnt, label_post_idx = _mask_summary(labels_valid_post) + loss_pre_cnt, loss_pre_idx = _mask_summary(loss_valid_pre) + loss_post_cnt, loss_post_idx = _mask_summary(loss_valid_post) + logger.info( + "[THD_DIAG][align] labels_pre_count=%d labels_post_count=%d labels_same=%s labels_pre_%s labels_post_%s", + label_pre_cnt, + label_post_cnt, + str(bool(torch.equal(labels_valid_pre, labels_valid_post))), + label_pre_idx, + label_post_idx, + ) + logger.info( + "[THD_DIAG][align] loss_pre_count=%d loss_post_count=%d loss_same=%s loss_pre_%s loss_post_%s", + loss_pre_cnt, + loss_post_cnt, + str(bool(torch.equal(loss_valid_pre, loss_valid_post))), + loss_pre_idx, + loss_post_idx, + ) + logger.info( + "[THD_DIAG][align] pre_label_loss_same=%s post_label_loss_same=%s", + str(bool(torch.equal(labels_valid_pre, loss_valid_pre))), + str(bool(torch.equal(labels_valid_post, loss_valid_post))), + ) _, _, vision_mask_thd = reorganize_inputs( input_ids=input_ids_thd, pixel_values=pixel_values, @@ -530,7 +603,7 @@ def forward( tmp_embeddings[vision_mask] = deepstack_visual_embed tmp_embeddings_thd = preprocess_packed_seqs( tmp_embeddings.contiguous(), - attention_mask, + attn_mask_bool, pre_process=True, pg_collection=self.pg_collection, )[0] @@ -542,7 +615,7 @@ def forward( combined_embeddings_thd = ( preprocess_packed_seqs( combined_embeddings.transpose(0, 1).contiguous(), - attention_mask, + attn_mask_bool, pre_process=True, pg_collection=self.pg_collection, )[0] @@ -562,9 +635,17 @@ def forward( if packed_seq_params is not None: if attention_mask is None: attention_mask = torch.ones_like(input_ids, dtype=torch.bool, device=input_ids.device) + attn_mask_bool = attention_mask.bool() lm_input_ids, _ = preprocess_packed_seqs( - input_ids, attention_mask, pre_process=True, pg_collection=self.pg_collection + input_ids, attn_mask_bool, pre_process=True, pg_collection=self.pg_collection ) + if moe_padding_mask_for_lm is not None: + moe_padding_mask_for_lm = preprocess_packed_seqs( + moe_padding_mask_for_lm.to(dtype=torch.int32), + attn_mask_bool, + pre_process=True, + pg_collection=self.pg_collection, + )[0].bool() visual_pos_masks = vision_mask deepstack_visual_embeds = deepstack_feature_lists @@ -591,38 +672,186 @@ def forward( ) if position_ids is None: - # BSHD - # Megatron uses 4D bool masks ([B|1,1,S,S], True=masked); HF uses 2D keep masks ([B,S], 1=keep) - # For simplicity, we set hf_attention_mask to None. - hf_attention_mask = None - position_ids, _ = get_rope_index( - self.config.spatial_merge_size, - self.image_token_id, - self.video_token_id, - self.vision_start_token_id, - input_ids, - image_grid_thw=image_grid_thw, - video_grid_thw=video_grid_thw, - attention_mask=hf_attention_mask, - ) # [3*b*s] if packed_seq_params is not None: - # convert position_ids to THD format + # Packed sequences: compute MRoPE per sub-sequence independently + # so that each sub-sequence's positions restart from 0. + cu = rope_cu_seqlens if rope_cu_seqlens is not None else packed_seq_params.cu_seqlens_q + if cu.dim() > 1: + cu = cu.squeeze() + seq_lens = (cu[1:] - cu[:-1]).tolist() + num_seqs = len(seq_lens) + max_subseq_len = int(max(seq_lens)) + total_len = input_ids.shape[1] + flat_ids = input_ids.squeeze(0) + + sub_ids = torch.zeros(num_seqs, max_subseq_len, dtype=input_ids.dtype, device=input_ids.device) + sub_mask = torch.zeros(num_seqs, max_subseq_len, dtype=torch.int32, device=input_ids.device) + for i, sl in enumerate(seq_lens): + start = int(cu[i].item()) + sl_int = int(sl) + sub_ids[i, :sl_int] = flat_ids[start : start + sl_int] + sub_mask[i, :sl_int] = 1 + + position_ids, _ = get_rope_index( + self.config.spatial_merge_size, + self.image_token_id, + self.video_token_id, + self.vision_start_token_id, + sub_ids, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + attention_mask=sub_mask, + ) # [3, num_seqs, max_subseq_len] + + packed_pos = torch.zeros(3, 1, total_len, dtype=position_ids.dtype, device=position_ids.device) + for i, sl in enumerate(seq_lens): + start = int(cu[i].item()) + sl_int = int(sl) + packed_pos[:, 0, start : start + sl_int] = position_ids[:, i, :sl_int] + + if thd_diag_mrope_enabled() and is_rank_0(): + # Summarize per-subsequence MRoPE position behavior before THD remap. + # This is diagnostics-only and intentionally does not affect semantics. + show_n = 4 + head_stats = [] + for i, sl in enumerate(seq_lens[:show_n]): + start = int(cu[i].item()) + sl_int = int(sl) + if sl_int <= 0: + head_stats.append(f"{i}:len=0") + continue + pos0 = packed_pos[0, 0, start : start + sl_int] + nonmono = int((pos0[1:] < pos0[:-1]).sum().item()) if sl_int > 1 else 0 + head_stats.append( + f"{i}:len={sl_int},start={int(pos0[0].item())},end={int(pos0[-1].item())}," + f"min={int(pos0.min().item())},max={int(pos0.max().item())},dec={nonmono}" + ) + logger.info( + "[THD_DIAG][mrope] pre_thd cp_size=%d num_seqs=%d total_len=%d seq_lens_head=%s seg_stats_head=%s", + int(cp_size), + int(num_seqs), + int(total_len), + seq_lens[:show_n], + "; ".join(head_stats), + ) + + position_ids = packed_pos + + attn_mask_bool = attention_mask.bool() position_ids = ( preprocess_packed_seqs( position_ids.permute(1, 2, 0), - attention_mask, + attn_mask_bool, pre_process=True, pg_collection=self.pg_collection, )[0] .permute(2, 0, 1) .contiguous() ) + if thd_diag_mrope_enabled() and is_rank_0(): + pos0_post = position_ids[0, 0] + post_len = int(pos0_post.numel()) + head_vals = pos0_post[:12].tolist() + tail_vals = pos0_post[-12:].tolist() if post_len > 12 else [] + nonmono_post = int((pos0_post[1:] < pos0_post[:-1]).sum().item()) if post_len > 1 else 0 + logger.info( + "[THD_DIAG][mrope] post_thd cp_size=%d len=%d min=%d max=%d dec=%d head=%s tail=%s", + int(cp_size), + post_len, + int(pos0_post.min().item()) if post_len > 0 else -1, + int(pos0_post.max().item()) if post_len > 0 else -1, + nonmono_post, + head_vals, + tail_vals, + ) + # Strict check: verify THD-remapped position_ids equals explicit bool-mask gather + # semantics for cp_size==1. This is diagnostics-only. + if int(cp_size) == 1: + tp_size = int(self.pg_collection.tp.size()) + align_size = tp_size + packed_pos_bsd = packed_pos.permute(1, 2, 0).contiguous() # [1, S, 3] + valid_len = int(attn_mask_bool[0].sum().item()) + padded_len = valid_len + ((align_size - (valid_len % align_size)) % align_size) + expected_bsd = torch.zeros( + (1, padded_len, packed_pos_bsd.size(-1)), + dtype=packed_pos_bsd.dtype, + device=packed_pos_bsd.device, + ) + expected_bsd[0, :valid_len] = packed_pos_bsd[0, attn_mask_bool[0]] + expected_pid = expected_bsd.permute(2, 0, 1).contiguous() + + same = bool(torch.equal(position_ids, expected_pid)) + mismatch = (position_ids != expected_pid).any(dim=0).squeeze(0) + mismatch_cnt = int(mismatch.sum().item()) + mismatch_idx = torch.nonzero(mismatch, as_tuple=False).flatten() + head_idx = mismatch_idx[:8].tolist() + tail_idx = mismatch_idx[-8:].tolist() if mismatch_idx.numel() > 8 else [] + logger.info( + "[THD_DIAG][mrope] strict_match=%s mismatch_count=%d expected_len=%d actual_len=%d mismatch_head=%s mismatch_tail=%s", + str(same), + mismatch_cnt, + int(expected_pid.size(-1)), + int(position_ids.size(-1)), + head_idx, + tail_idx, + ) + else: + logger.info( + "[THD_DIAG][mrope] strict_match_skipped cp_size=%d (currently only checks cp_size==1)", + int(cp_size), + ) attention_mask = None self.language_model.rotary_pos_emb.is_thd_format = True + else: + # BSHD + # Megatron uses 4D bool masks ([B|1,1,S,S], True=masked); HF uses 2D keep masks ([B,S], 1=keep) + # For get_rope_index we pass None to avoid semantic mismatch. + hf_attention_mask = None + position_ids, _ = get_rope_index( + self.config.spatial_merge_size, + self.image_token_id, + self.video_token_id, + self.vision_start_token_id, + input_ids, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + attention_mask=hf_attention_mask, + ) # [3*b*s] torch.cuda.nvtx.range_pop() torch.cuda.nvtx.range_push("Qwen3VLModel.forward.language_model") + # For THD packed path we intentionally keep attention_mask=None for model forward. + # MoE aux/global-aux accounting still needs to ignore tail padding tokens, so pass + # a dedicated padding_mask (True=padding) to GPTModel/Router. + padding_mask_for_moe = None + if packed_seq_params is not None and lm_input_ids is not None: + if moe_padding_mask_for_lm is not None: + padding_mask_for_moe = moe_padding_mask_for_lm.bool() + mask_source = "explicit" + else: + # Fallback for old call sites that do not provide explicit packed padding mask. + padding_mask_for_moe = lm_input_ids.eq(0) + mask_source = "token_eq_0_fallback" + if thd_diag_enabled() and is_rank_0(): + input_zero_cnt = int(input_ids.eq(0).sum().item()) if input_ids is not None else -1 + lm_zero_cnt = int(lm_input_ids.eq(0).sum().item()) + pad_cnt = int(padding_mask_for_moe.sum().item()) + tok_cnt = int(padding_mask_for_moe.numel()) + loss_valid = int((loss_mask > 0).sum().item()) if loss_mask is not None else -1 + loss_total = int(loss_mask.numel()) if loss_mask is not None else -1 + logger.info( + "[THD_DIAG][model] padding_mask_for_moe: source=%s total_tokens=%d padding_tokens=%d valid_tokens=%d input_zero_tokens=%d lm_zero_tokens=%d loss_valid_tokens=%d loss_total_tokens=%d", + mask_source, + tok_cnt, + pad_cnt, + tok_cnt - pad_cnt, + input_zero_cnt, + lm_zero_cnt, + loss_valid, + loss_total, + ) + output = self.language_model( input_ids=lm_input_ids, position_ids=position_ids, # None in encoder @@ -630,8 +859,9 @@ def forward( decoder_input=combined_embeddings, # only not None in the first decoder PP stage labels=labels, # only not None in the last decoder PP stage loss_mask=loss_mask, # Added for THD training compatibility - inference_params=inference_params, # currently always None - packed_seq_params=packed_seq_params, # currently always None + padding_mask=padding_mask_for_moe, # for MoE routing/aux-loss token accounting + inference_params=inference_params, # training path keeps this as None + packed_seq_params=packed_seq_params, visual_pos_masks=visual_pos_masks, deepstack_visual_embeds=deepstack_visual_embeds, **(extra_block_kwargs or {}), diff --git a/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/text_model.py b/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/text_model.py index d067655a1b..d6781ced3a 100644 --- a/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/text_model.py +++ b/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/text_model.py @@ -120,6 +120,7 @@ def forward( *, inference_params: Optional[BaseInferenceContext] = None, loss_mask: Optional[Tensor] = None, + padding_mask: Optional[Tensor] = None, # args for deepstack visual_pos_masks: Optional[torch.Tensor] = None, deepstack_visual_embeds: Optional[list[torch.Tensor]] = None, @@ -148,6 +149,7 @@ def forward( decoder_input=decoder_input, inference_context=inference_context, packed_seq_params=packed_seq_params, + padding_mask=padding_mask, ) ( @@ -156,7 +158,8 @@ def forward( rotary_pos_cos, rotary_pos_sin, sequence_len_offset, - ) = preproc_output[:5] + padding_mask, + ) = preproc_output[:6] # Run decoder. hidden_states = self.decoder( @@ -170,6 +173,7 @@ def forward( # the standard components only. packed_seq_params=packed_seq_params, sequence_len_offset=sequence_len_offset, + padding_mask=padding_mask, visual_pos_masks=visual_pos_masks, deepstack_visual_embeds=deepstack_visual_embeds, **(extra_block_kwargs or {}), @@ -204,6 +208,7 @@ def _sp_scatter_embedding(input_ids, position_ids): loss_mask=loss_mask, decoder_input=decoder_input, attention_mask=attention_mask, + padding_mask=padding_mask, inference_params=inference_params, packed_seq_params=packed_seq_params, sequence_len_offset=sequence_len_offset, diff --git a/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/transformer_block.py b/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/transformer_block.py index 6284617bed..440e45c873 100644 --- a/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/transformer_block.py +++ b/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/transformer_block.py @@ -503,6 +503,7 @@ def _checkpointed_forward( rotary_pos_emb: Tensor, attention_bias: Tensor, packed_seq_params: PackedSeqParams, + padding_mask: Optional[Tensor], use_inner_fp8_context: bool, # args for deepstack visual_pos_masks: Optional[torch.Tensor] = None, @@ -517,6 +518,7 @@ def custom_forward( context, context_mask, rotary_pos_emb, + padding_mask, visual_pos_masks, *deepstack_visual_embeds_args, ): @@ -538,6 +540,7 @@ def custom_forward( attention_bias=attention_bias, inference_context=None, packed_seq_params=packed_seq_params, + padding_mask=padding_mask, ) if self.pre_process and deepstack_visual_embeds is not None: @@ -567,6 +570,7 @@ def checkpoint_handler(forward_func): context, context_mask, rotary_pos_emb, + padding_mask, visual_pos_masks, *deepstack_visual_embeds_tuple, ) @@ -579,6 +583,7 @@ def checkpoint_handler(forward_func): context, context_mask, rotary_pos_emb, + padding_mask, visual_pos_masks, *deepstack_visual_embeds_tuple, ) @@ -618,6 +623,7 @@ def checkpoint_handler(forward_func): context, context_mask, rotary_pos_emb, + padding_mask, visual_pos_masks, *deepstack_visual_embeds_tuple, ) @@ -639,6 +645,7 @@ def forward( inference_context: Optional[BaseInferenceContext] = None, packed_seq_params: Optional[PackedSeqParams] = None, sequence_len_offset: Optional[Tensor] = None, + padding_mask: Optional[Tensor] = None, *, inference_params: Optional[BaseInferenceContext] = None, # args for deepstack @@ -730,6 +737,7 @@ def forward( rotary_pos_emb=rotary_pos_emb, attention_bias=attention_bias, packed_seq_params=packed_seq_params, + padding_mask=padding_mask, use_inner_fp8_context=use_inner_fp8_context, visual_pos_masks=visual_pos_masks, deepstack_visual_embeds=deepstack_visual_embeds, @@ -754,6 +762,7 @@ def forward( inference_context=inference_context, packed_seq_params=packed_seq_params, sequence_len_offset=sequence_len_offset, + padding_mask=padding_mask, ) if self.pre_process and deepstack_visual_embeds is not None: diff --git a/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/utils.py b/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/utils.py index 35b40d3ee3..9f87bb96a0 100644 --- a/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/utils.py +++ b/src/megatron/bridge/models/qwen_vl/modelling_qwen3_vl/utils.py @@ -13,6 +13,7 @@ # limitations under the License. +import os from dataclasses import dataclass from typing import Optional, Sequence, Union @@ -26,6 +27,22 @@ from torch import nn from megatron.bridge.models.qwen_vl.modelling_qwen3_vl.transformer_config import Qwen3VLTransformerConfig +from megatron.bridge.utils.common_utils import is_rank_0 # noqa: F401 + + +def thd_diag_enabled() -> bool: + """Return whether THD diagnostics are enabled.""" + return os.environ.get("THD_DIAG", "0") not in ("0", "", "false", "False") + + +def thd_diag_align_enabled() -> bool: + """Return whether THD alignment diagnostics are enabled.""" + return os.environ.get("THD_DIAG_ALIGN", "0") not in ("0", "", "false", "False") + + +def thd_diag_mrope_enabled() -> bool: + """Return whether THD MRoPE diagnostics are enabled.""" + return os.environ.get("THD_DIAG_MROPE", "0") not in ("0", "", "false", "False") # copied from https://github.com/huggingface/transformers/blob/main/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -694,6 +711,11 @@ def preprocess_packed_seqs( See https://github.com/NVIDIA/TransformerEngine/issues/1368 """ batch_size = input_ids.shape[0] + # Boolean mask is required for advanced indexing semantics below: + # input_ids[i, attention_mask[i]] + # If mask is int (0/1), PyTorch treats it as positional index selection + # rather than keep-mask, which can silently corrupt THD remapping. + attention_mask = attention_mask.bool() # Ensure boolean dtype for correct advanced indexing (bool → mask select, # int → fancy index which silently corrupts data when values are 0/1). diff --git a/src/megatron/bridge/models/qwen_vl/qwen3_vl_step.py b/src/megatron/bridge/models/qwen_vl/qwen3_vl_step.py index c87202262b..1a044a06dd 100644 --- a/src/megatron/bridge/models/qwen_vl/qwen3_vl_step.py +++ b/src/megatron/bridge/models/qwen_vl/qwen3_vl_step.py @@ -36,6 +36,7 @@ logger = logging.getLogger(__name__) +_BATCH_LEVEL_PACKING_WARNED = False def get_batch_from_iterator( @@ -248,7 +249,19 @@ def forward_step( # To be compatible with qwen3vl, we move the sequence padding and packing to forward_step function. # Qwen3VL model need the original input and do cp and sp split in model.forward. - pack_sequences_in_batch = getattr(state.cfg.dataset, "pack_sequences_in_batch", False) + in_batch_pack_enabled = getattr(state.cfg.dataset, "pack_sequences_in_batch", False) + batch_level_pack_enabled = getattr(state.cfg.dataset, "batch_level_packing", False) + # qwen3_vl_step does not yet wire full THD metadata (rope/moe padding-mask) + # for batch-level packing. Keep this path disabled to avoid incorrect token + # accounting until parity with training.vlm_step is implemented. + global _BATCH_LEVEL_PACKING_WARNED + if batch_level_pack_enabled and not _BATCH_LEVEL_PACKING_WARNED: + logger.warning( + "dataset.batch_level_packing is not fully supported in qwen3_vl_step yet; " + "falling back to in-batch packing behavior." + ) + _BATCH_LEVEL_PACKING_WARNED = True + enable_sequence_packing = in_batch_pack_enabled tokens, labels, loss_mask, attention_mask, position_ids, packed_seq_params = pack_or_pad_batch_sequences( tokens, @@ -275,7 +288,7 @@ def forward_step( forward_args["input_ids"] = original_tokens # calculate position_ids in model forward forward_args["position_ids"] = None - if pack_sequences_in_batch: + if enable_sequence_packing: if forward_args["labels"] is not None: # When using pp, labels could be None forward_args["labels"] = forward_args["labels"].reshape(1, -1) diff --git a/src/megatron/bridge/recipes/__init__.py b/src/megatron/bridge/recipes/__init__.py index 46d51fdb64..3685045ec7 100644 --- a/src/megatron/bridge/recipes/__init__.py +++ b/src/megatron/bridge/recipes/__init__.py @@ -18,8 +18,18 @@ This module exposes all recipe configurations from all model families. """ -from megatron.bridge.diffusion.recipes.flux.flux import * -from megatron.bridge.diffusion.recipes.wan.wan import * +# Diffusion recipes require optional dependencies (e.g. diffusers). Keep +# non-diffusion training paths (LLM/VLM) usable when those deps are absent. +try: + from megatron.bridge.diffusion.recipes.flux.flux import * +except ModuleNotFoundError: + pass + +try: + from megatron.bridge.diffusion.recipes.wan.wan import * +except ModuleNotFoundError: + pass + from megatron.bridge.recipes.deepseek import * from megatron.bridge.recipes.gemma import * from megatron.bridge.recipes.gemma3_vl import * diff --git a/src/megatron/bridge/recipes/qwen_vl/__init__.py b/src/megatron/bridge/recipes/qwen_vl/__init__.py index 9bc536c45d..93dc981f59 100644 --- a/src/megatron/bridge/recipes/qwen_vl/__init__.py +++ b/src/megatron/bridge/recipes/qwen_vl/__init__.py @@ -52,6 +52,8 @@ qwen35_vl_35b_a3b_peft_config, qwen35_vl_35b_a3b_pretrain_mock_config, qwen35_vl_35b_a3b_sft_config, + qwen35_vl_35b_a3b_sft_energon_config, + qwen35_vl_35b_a3b_sft_energon_nopack_config, qwen35_vl_122b_a10b_peft_config, qwen35_vl_122b_a10b_pretrain_mock_config, qwen35_vl_122b_a10b_sft_config, @@ -77,6 +79,8 @@ "qwen35_vl_27b_sft_config", # Qwen3.5-VL SFT configs — MoE "qwen35_vl_35b_a3b_sft_config", + "qwen35_vl_35b_a3b_sft_energon_config", + "qwen35_vl_35b_a3b_sft_energon_nopack_config", "qwen35_vl_35b_a3b_fsdp_sft_config", "qwen35_vl_122b_a10b_sft_config", "qwen35_vl_397b_a17b_sft_config", diff --git a/src/megatron/bridge/recipes/qwen_vl/data/energon/task_encoder.py b/src/megatron/bridge/recipes/qwen_vl/data/energon/task_encoder.py index 50178a0749..12ec6d9f15 100644 --- a/src/megatron/bridge/recipes/qwen_vl/data/energon/task_encoder.py +++ b/src/megatron/bridge/recipes/qwen_vl/data/energon/task_encoder.py @@ -12,16 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import bisect import dataclasses import logging +import os import re from collections import defaultdict -from dataclasses import dataclass -from typing import Dict, List +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Union import numpy as np import torch -from megatron.energon import Batch, DefaultTaskEncoder +from megatron.energon import Batch, DefaultTaskEncoder, stateless from transformers import BatchEncoding from megatron.bridge.data.energon.task_encoder_utils import ( @@ -39,6 +41,69 @@ from megatron.bridge.training.utils.visual_inputs import Qwen2_5_VLVisualInputs +logger = logging.getLogger(__name__) +_LOW_CONTENT_LOG_COUNT = 0 + + +def _thd_diag_enabled() -> bool: + return os.environ.get("THD_DIAG", "0") not in ("0", "", "false", "False") + + +def _thd_diag_low_content_threshold() -> int: + raw = os.environ.get("THD_DIAG_LOW_CONTENT_TOKENS", "2048") + try: + return max(0, int(raw)) + except ValueError: + return 2048 + + +def _thd_diag_max_low_content_logs() -> int: + raw = os.environ.get("THD_DIAG_MAX_LOW_CONTENT_LOGS", "100") + try: + return max(0, int(raw)) + except ValueError: + return 100 + + +def _search_for_fit(numbers: List[int], capacity: int) -> int: + """Binary search for the largest number that fits within capacity.""" + index = bisect.bisect(numbers, capacity) + return -1 if index == 0 else (index - 1) + + +def greedy_knapsack(item_sizes: List[int], samples: List, max_capacity: int) -> List: + """Greedy bin-packing with binary search. + + Sorts samples by length ascending, then greedily fills each bin by picking + the largest item that still fits (via binary search). Returns a list of bins, + each bin being a list of samples. + """ + assert len(item_sizes) == len(samples) + if not item_sizes: + return [] + + sorted_sizes, sorted_samples = zip(*sorted(zip(item_sizes, samples), key=lambda x: x[0])) + sorted_sizes = list(sorted_sizes) + sorted_samples = list(sorted_samples) + + if sorted_sizes[-1] > max_capacity: + raise ValueError(f"Sample size {sorted_sizes[-1]} exceeds max_capacity {max_capacity}") + + knapsacks = [] + while sorted_sizes: + current_knapsack = [] + remaining = max_capacity + while True: + idx = _search_for_fit(sorted_sizes, remaining) + if idx == -1: + break + remaining -= sorted_sizes[idx] + sorted_sizes.pop(idx) + current_knapsack.append(sorted_samples.pop(idx)) + knapsacks.append(current_knapsack) + return knapsacks + + def process_vision( processor, images, videos, fps=None, model_version: str = "qwen-vl", min_pixels=None, max_pixels=None ): @@ -71,19 +136,25 @@ def process_vision( def _resolve_hf_mm_token_ids(hf_tokenizer): - """Resolve HF tokenizer ids for and