From 6d21c24a1712a7e023c755fc6382f826662e6158 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Wed, 22 Apr 2026 11:45:36 -0700 Subject: [PATCH] [recipe] feat: enable THD packing by default for Qwen3.5-VL finetune Flip pack_sequences_in_batch from False to True in the shared _qwen35_vl_apply_common helper used by all Qwen3.5-VL SFT and PEFT recipes. When PP>1 or EP>1 the step function already forces each micro-batch to pad to seq_length; building THD cu_seqlens lets the attention kernel skip padding FLOPs on short samples. Pretrain recipes are unaffected (they route through _qwen3_vl_common). Signed-off-by: Chen Cui --- src/megatron/bridge/recipes/qwen_vl/qwen35_vl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/megatron/bridge/recipes/qwen_vl/qwen35_vl.py b/src/megatron/bridge/recipes/qwen_vl/qwen35_vl.py index af089cc28bf..da2eb21d083 100644 --- a/src/megatron/bridge/recipes/qwen_vl/qwen35_vl.py +++ b/src/megatron/bridge/recipes/qwen_vl/qwen35_vl.py @@ -125,7 +125,10 @@ def _qwen35_vl_apply_common( # Dataset configuration cfg.dataset.seq_length = 4096 cfg.dataset.hf_processor_path = hf_path - cfg.dataset.pack_sequences_in_batch = False + # Enable THD packing by default for SFT/PEFT: the step function builds + # cu_seqlens so attention skips padding FLOPs on short samples (especially + # when PP/EP forces micro-batches to pad to seq_length). + cfg.dataset.pack_sequences_in_batch = True # DDP settings cfg.ddp.overlap_grad_reduce = False