From 88cf0c1017ed66b67f2eb100b8eb48e5384a90de Mon Sep 17 00:00:00 2001 From: Aditya Ghai Date: Mon, 17 Aug 2026 23:01:13 +0530 Subject: [PATCH 1/2] Forward rope_scaling_factor to LLaVAModel in the VLM builders Both VLM call sites pass language_rope_scaling but never language_rope_scaling_factor, so LLaVAModel falls back to its default of 8.0 and --rope-scaling-factor is silently ignored when training a VLM. LLaVAModel already accepts the parameter and forwards it to the language model, so only the call sites need updating. Reported for the GPT path in #6305, which notes that the VLM call sites are a separate surface not covered by its linked PR. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Aditya Ghai --- examples/multimodal/model.py | 1 + pretrain_vlm.py | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/multimodal/model.py b/examples/multimodal/model.py index a2d83428338..073716e5da4 100644 --- a/examples/multimodal/model.py +++ b/examples/multimodal/model.py @@ -217,6 +217,7 @@ def model_provider( patch_dim=args.patch_dim, language_rotary_base=args.rotary_base, language_rope_scaling=args.use_rope_scaling, + language_rope_scaling_factor=args.rope_scaling_factor, hybrid_layer_pattern=args.hybrid_layer_pattern, fp16_lm_cross_entropy=args.fp16_lm_cross_entropy, image_token_index=image_token_index, diff --git a/pretrain_vlm.py b/pretrain_vlm.py index f29d030c479..6bc0c8c3094 100644 --- a/pretrain_vlm.py +++ b/pretrain_vlm.py @@ -206,6 +206,7 @@ def model_provider( language_position_embedding_type=args.position_embedding_type, language_rotary_percent=args.rotary_percent, language_rope_scaling=args.use_rope_scaling, + language_rope_scaling_factor=args.rope_scaling_factor, pre_process=parallel_state.is_pipeline_first_stage(), post_process=parallel_state.is_pipeline_last_stage(), add_encoder=parallel_state.is_pipeline_first_stage(), From 187a7596c6be36c97eac96b9926a34d109dda50d Mon Sep 17 00:00:00 2001 From: Aditya Ghai Date: Mon, 17 Aug 2026 23:01:26 +0530 Subject: [PATCH 2/2] Forward rotary_base to LLaVAModel in pretrain_vlm.py pretrain_vlm.py never passes language_rotary_base, so LLaVAModel falls back to its default of 10000 and --rotary-base is silently ignored. The sibling call site in examples/multimodal/model.py already passes it. Unlike RoPE scaling, rotary_base is not opt-in: it is used by every RoPE model on this path, and it must match the pretrained checkpoint (Llama 3.x uses 500000). Fixes #6585 Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Aditya Ghai --- pretrain_vlm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pretrain_vlm.py b/pretrain_vlm.py index 6bc0c8c3094..678a34ca5ba 100644 --- a/pretrain_vlm.py +++ b/pretrain_vlm.py @@ -205,6 +205,7 @@ def model_provider( parallel_output=parallel_output, language_position_embedding_type=args.position_embedding_type, language_rotary_percent=args.rotary_percent, + language_rotary_base=args.rotary_base, language_rope_scaling=args.use_rope_scaling, language_rope_scaling_factor=args.rope_scaling_factor, pre_process=parallel_state.is_pipeline_first_stage(),