cp: feat: default packed_sequence to True in all finetune recipes (2284) into r0.3.0

src/megatron/bridge/recipes/gemma/gemma2.py

@@ -340,7 +340,7 @@ def _gemma2_finetune_common(
     # Finetuning-specific params
     pretrained_checkpoint: Optional[str] = None,
     peft: Union[str, PEFT, None] = "lora",
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training params
     train_iters: int = 100,
     global_batch_size: Optional[int] = None,

src/megatron/bridge/recipes/gemma/gemma3.py

@@ -376,7 +376,7 @@ def _gemma3_finetune_common(
     name: str = "default",
     # Finetuning-specific
     pretrained_checkpoint: str | None = None,
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training hyperparameters
     train_iters: int = 100,
     global_batch_size: int | None = None,

src/megatron/bridge/recipes/glm/glm45.py

@@ -369,6 +369,7 @@ def glm45_355b_finetune_config(**user_kwargs: Unpack[GLM45FinetuneKwargs]) -> Co
         "expert_model_parallel_size": 16 if is_full_sft else 4,
         "peft": peft_value,
         "finetune_lr": 5e-6 if is_full_sft else 1e-4,
+        "packed_sequence": False,  # Packed sequence is not supported for GLM 4.5
     }
     kwargs: GLM45FinetuneKwargs = {**recommended, **user_kwargs}
     return _glm45_finetune_common(**kwargs)
@@ -391,6 +392,7 @@ def glm45_air_106b_finetune_config(**user_kwargs: Unpack[GLM45FinetuneKwargs]) -
         "expert_model_parallel_size": 8 if is_full_sft else 4,
         "peft": peft_value,
         "finetune_lr": 5e-6 if is_full_sft else 1e-4,
+        "packed_sequence": False,  # Packed sequence is not supported for GLM 4.5
     }
     kwargs: GLM45FinetuneKwargs = {**recommended, **user_kwargs}
     return _glm45_finetune_common(**kwargs)
@@ -412,7 +414,7 @@ def _glm45_finetune_common(
     # Finetuning-specific params
     pretrained_checkpoint: Optional[str] = None,
     peft: Optional[Union[str, PEFT]] = "lora",
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training params
     train_iters: int = 1000,
     global_batch_size: int = 128,

src/megatron/bridge/recipes/gpt_oss/gpt_oss.py

@@ -380,7 +380,7 @@ def _gpt_oss_finetune_common(
     # Finetuning-specific params
     pretrained_checkpoint: Optional[str] = None,
     peft: Optional[Union[str, PEFT]] = "lora",
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training params
     train_iters: int = 1000,
     global_batch_size: int = 128,

src/megatron/bridge/recipes/llama/llama3.py

@@ -922,7 +922,7 @@ def _llama3_finetune_common(
     name: str = "default",
     # Finetuning-specific params
     pretrained_checkpoint: str | None = None,
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training params
     train_iters: int = 1000,
     global_batch_size: int | None = None,

src/megatron/bridge/recipes/moonlight/moonlight_16b.py

@@ -519,7 +519,7 @@ def _moonlight_finetune_common(
     # Finetuning-specific params
     pretrained_checkpoint: Optional[str] = None,
     peft: Optional[Union[str, PEFT]] = "lora",
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training params
     train_iters: int = 1000,
     global_batch_size: int = 128,

src/megatron/bridge/recipes/nemotronh/nemotron_3_nano.py

@@ -399,7 +399,7 @@ def _nemotron_3_nano_finetune_common(
     # Finetuning-specific params
     pretrained_checkpoint: Optional[str] = None,
     peft: Optional[Union[str, PEFT]] = "lora",
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training params
     train_iters: int = 1000,
     global_batch_size: int = 128,

src/megatron/bridge/recipes/nemotronh/nemotronh.py

@@ -470,7 +470,7 @@ def _nemotronh_finetune_common(
     # Finetuning-specific params
     pretrained_checkpoint: str | None = None,
     peft: str | PEFT | None = "lora",
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training params
     train_iters: int = 1000,
     global_batch_size: int = 128,
@@ -508,7 +508,7 @@ def _nemotronh_finetune_common(
         sequence_parallelism: Whether to use sequence parallelism.
         pretrained_checkpoint: Path to pretrained checkpoint to load from.
         peft: PEFT configuration (e.g., "lora", "dora") or PEFT object. None for full SFT. Default: "lora".
-        packed_sequence: Whether to use packed sequences. Default: False.
+        packed_sequence: Whether to use packed sequences. Default: True.
         train_iters: Total number of training iterations. Default: 1000.
         global_batch_size: Global batch size. Default: 128.
         micro_batch_size: Micro batch size. Default: 1.

src/megatron/bridge/recipes/olmoe/olmoe_7b.py

@@ -495,7 +495,7 @@ def _olmoe_finetune_common(
     # Finetuning-specific params
     pretrained_checkpoint: Optional[str] = None,
     peft: Optional[Union[str, PEFT]] = "lora",
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training params
     train_iters: int = 1000,
     global_batch_size: int = 128,

src/megatron/bridge/recipes/qwen/qwen2.py

@@ -594,7 +594,7 @@ def _qwen2_finetune_common(
     # Finetuning-specific params
     pretrained_checkpoint: Optional[str] = None,
     peft: Union[str, PEFT, None] = "lora",
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training params
     train_iters: int = 100,
     global_batch_size: Optional[int] = None,

src/megatron/bridge/recipes/qwen/qwen3.py

@@ -511,7 +511,7 @@ def _qwen3_finetune_common(
     # Finetuning-specific params
     pretrained_checkpoint: str | None = None,
     peft: str | PEFT | None = "lora",
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training params
     train_iters: int = 1000,
     global_batch_size: int | None = None,  # Auto-select based on packed_sequence if None

src/megatron/bridge/recipes/qwen/qwen3_moe.py

@@ -472,7 +472,7 @@ def _qwen3_moe_finetune_common(
     name: str = "default",
     # Finetuning-specific
     pretrained_checkpoint: Optional[str] = None,
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training hyperparameters
     train_iters: int = 100,
     global_batch_size: Optional[int] = None,

src/megatron/bridge/recipes/qwen/qwen3_next.py

@@ -378,6 +378,7 @@ def qwen3_next_80b_a3b_finetune_config(**user_kwargs: Unpack[Qwen3NextFinetuneKw
         "finetune_lr": 5e-6,
         "min_lr": 5e-6,
         "enable_recompute": True,
+        "packed_sequence": False,  # Sequence packing is not supported for Qwen3-Next
     }
     combined_kwargs: Qwen3NextFinetuneKwargs = {**recommended_kwargs, **user_kwargs}
     config = _qwen3_next_finetune_common(**combined_kwargs)
@@ -405,7 +406,7 @@ def _qwen3_next_finetune_common(
     # Finetuning-specific params
     pretrained_checkpoint: str | None = None,
     peft: str | PEFT | None = None,
-    packed_sequence: bool = False,
+    packed_sequence: bool = True,
     # Training params
     train_iters: int = 1000,
     global_batch_size: int | None = None,  # Auto-select based on packed_sequence if None

src/megatron/bridge/recipes/utils/finetune_utils.py

@@ -50,7 +50,7 @@ def default_peft_config(peft_scheme: str | PEFT | None, **kwargs) -> PEFT | None
     raise ValueError(f"Invalid peft type: {type(peft_scheme)}. Expected str, PEFT instance, or None")
 
 
-def default_squad_config(seq_length: int, packed_sequence: bool = False, pad_seq_to_mult: int = 1) -> HFDatasetConfig:
+def default_squad_config(seq_length: int, packed_sequence: bool = True, pad_seq_to_mult: int = 1) -> HFDatasetConfig:
     """Create default SQuAD dataset configuration for finetuning recipes.
 
     Args:

tests/unit_tests/recipes/test_qwen_recipes.py

@@ -98,6 +98,7 @@ class _FakeModelCfg:
 
     def __init__(self):
         self.cross_entropy_fusion_impl = "native"
+        self.context_parallel_size = 1
 
     def finalize(self):
         # qwen3 recipe may call finalize(); make it a no-op