diff --git a/examples/multimodal/model.py b/examples/multimodal/model.py
index a2d83428338..073716e5da4 100644
--- a/examples/multimodal/model.py
+++ b/examples/multimodal/model.py
@@ -217,6 +217,7 @@ def model_provider(
         patch_dim=args.patch_dim,
         language_rotary_base=args.rotary_base,
         language_rope_scaling=args.use_rope_scaling,
+        language_rope_scaling_factor=args.rope_scaling_factor,
         hybrid_layer_pattern=args.hybrid_layer_pattern,
         fp16_lm_cross_entropy=args.fp16_lm_cross_entropy,
         image_token_index=image_token_index,
diff --git a/pretrain_vlm.py b/pretrain_vlm.py
index f29d030c479..678a34ca5ba 100644
--- a/pretrain_vlm.py
+++ b/pretrain_vlm.py
@@ -205,7 +205,9 @@ def model_provider(
         parallel_output=parallel_output,
         language_position_embedding_type=args.position_embedding_type,
         language_rotary_percent=args.rotary_percent,
+        language_rotary_base=args.rotary_base,
         language_rope_scaling=args.use_rope_scaling,
+        language_rope_scaling_factor=args.rope_scaling_factor,
         pre_process=parallel_state.is_pipeline_first_stage(),
         post_process=parallel_state.is_pipeline_last_stage(),
         add_encoder=parallel_state.is_pipeline_first_stage(),