From d18c83cea6e2ba78ba154e926bf05e3092161f9a Mon Sep 17 00:00:00 2001 From: faresobeid <111092724+faresobeid@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:07:00 +0000 Subject: [PATCH] chore(configs)!: drop vlm attr-path fields and the unread orchestrator.model.vlm --- .../ci/nightly/multimodal_color_codeword.toml | 2 -- docs/advanced.md | 14 ++++---- examples/extra/vlm/sft-moe.toml | 2 -- examples/extra/vlm/sft.toml | 2 -- .../src/prime_rl/configs/shared.py | 9 ----- .../src/prime_rl/configs/trainer.py | 4 +++ .../src/prime_rl/utils/validation.py | 6 +--- tests/unit/test_configs.py | 35 +------------------ 8 files changed, 12 insertions(+), 62 deletions(-) diff --git a/configs/ci/nightly/multimodal_color_codeword.toml b/configs/ci/nightly/multimodal_color_codeword.toml index 46999258bbd..9e2050d3711 100644 --- a/configs/ci/nightly/multimodal_color_codeword.toml +++ b/configs/ci/nightly/multimodal_color_codeword.toml @@ -10,8 +10,6 @@ num_infer_gpus = 4 name = "Qwen/Qwen3.5-4B" [model.vlm] -vision_encoder_attr = "model.visual" -language_model_attr = "model.language_model" [orchestrator] batch_size = 256 diff --git a/docs/advanced.md b/docs/advanced.md index ec6d7c43abd..eb4954026c7 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -142,10 +142,12 @@ The experimental `shard_fused_on_dim1 = true` shards fused 2-D weights along dim The built-in VLM registry covers: -| Family | `model_type` | Vision attr | LM attr | -|---|---|---|---| -| Qwen3.5 | `qwen3_5` | `model.visual` | `model.language_model` | -| Qwen3.5-MoE | `qwen3_5_moe` | `model.visual` | `model.language_model` | +| Family | `model_type` | +|---|---| +| Qwen3.5 | `qwen3_5` | +| Qwen3.5-MoE | `qwen3_5_moe` | + +The trainer finds the vision encoder at `model.visual` and the language model at `model.language_model`. ### Enabling VLM Mode @@ -158,13 +160,9 @@ optimization_dtype = "bfloat16" reduce_dtype = "bfloat16" [model.vlm] -vision_encoder_attr = "model.visual" -language_model_attr = "model.language_model" # freeze_vision_encoder = true # default; set false to fine-tune the encoder ``` -The weight-broadcast key prefix is derived as `{language_model_attr}.layers.` automatically. - VLM training requires a registered custom PrimeRL implementation. ### Limitations diff --git a/examples/extra/vlm/sft-moe.toml b/examples/extra/vlm/sft-moe.toml index 94ff1f52a5c..e37f9ea4dce 100644 --- a/examples/extra/vlm/sft-moe.toml +++ b/examples/extra/vlm/sft-moe.toml @@ -13,8 +13,6 @@ ep = 8 # keep in sync with deployment.num_train_gpus name = "auto" [model.vlm] -vision_encoder_attr = "model.visual" -language_model_attr = "model.language_model" freeze_vision_encoder = true [model.lora] diff --git a/examples/extra/vlm/sft.toml b/examples/extra/vlm/sft.toml index 5b6511553b8..b70ff9aa968 100644 --- a/examples/extra/vlm/sft.toml +++ b/examples/extra/vlm/sft.toml @@ -11,8 +11,6 @@ reduce_dtype = "bfloat16" name = "auto" [model.vlm] -vision_encoder_attr = "model.visual" -language_model_attr = "model.language_model" freeze_vision_encoder = true [model.lora] diff --git a/packages/prime-rl-configs/src/prime_rl/configs/shared.py b/packages/prime-rl-configs/src/prime_rl/configs/shared.py index 335b706fea7..9dd7fece371 100644 --- a/packages/prime-rl-configs/src/prime_rl/configs/shared.py +++ b/packages/prime-rl-configs/src/prime_rl/configs/shared.py @@ -183,12 +183,6 @@ def resolve_project_dir(self): class VLMConfig(BaseConfig): - vision_encoder_attr: str - """Dotted attribute path to the vision encoder module (e.g. ``model.visual``).""" - - language_model_attr: str - """Dotted attribute path to the language model module (e.g. ``model.language_model``).""" - freeze_vision_encoder: bool = True """Freeze the vision encoder parameters during training.""" @@ -200,9 +194,6 @@ class BaseModelConfig(BaseConfig): trust_remote_code: bool = False """Trust remote code when initializing the tokenizer.""" - vlm: "VLMConfig | None" = None - """VLM configuration. Setting this enables vision-language model support.""" - class DynamoConfig(BaseConfig): enabled: bool = True diff --git a/packages/prime-rl-configs/src/prime_rl/configs/trainer.py b/packages/prime-rl-configs/src/prime_rl/configs/trainer.py index 22f741cd27a..17985732df2 100644 --- a/packages/prime-rl-configs/src/prime_rl/configs/trainer.py +++ b/packages/prime-rl-configs/src/prime_rl/configs/trainer.py @@ -15,6 +15,7 @@ ResumeConfig, TrainerLogConfig, TransportConfig, + VLMConfig, WeightBroadcastConfig, ZMQTransportConfig, ) @@ -267,6 +268,9 @@ class MoERuntimeConfig(BaseConfig): class ModelConfig(BaseModelConfig): + vlm: VLMConfig | None = None + """VLM configuration. Setting this enables vision-language model support.""" + conversion_dir: Path | None = None """Directory for the auto-converted weights (written to a `prime`/`hf` subdirectory). If not set, we write into the model snapshot directory.""" diff --git a/packages/prime-rl-configs/src/prime_rl/utils/validation.py b/packages/prime-rl-configs/src/prime_rl/utils/validation.py index fce93050f2b..1f5ff3b6942 100644 --- a/packages/prime-rl-configs/src/prime_rl/utils/validation.py +++ b/packages/prime-rl-configs/src/prime_rl/utils/validation.py @@ -71,11 +71,7 @@ def propagate(shared_path: str, *targets: str) -> None: "inference.vllm.model", "orchestrator.model.name", ) - propagate( - "model.vlm", - "trainer.model.vlm", - "orchestrator.model.vlm", - ) + propagate("model.vlm", "trainer.model.vlm") # [log] propagate("log.level", "trainer.log.level", "orchestrator.log.level", "inference.log.level") diff --git a/tests/unit/test_configs.py b/tests/unit/test_configs.py index 9c4851132fe..6aefb8921fa 100644 --- a/tests/unit/test_configs.py +++ b/tests/unit/test_configs.py @@ -590,46 +590,13 @@ def test_multi_node_auto_inference_parallelism(): assert config.inference.vllm.data_parallel_size == 2 -def test_orchestrator_vlm_requires_renderer(): - with pytest.raises(ValidationError, match="renderer"): - OrchestratorConfig.model_validate( - { - "model": { - "name": "Qwen/Qwen3-VL-4B-Instruct", - "vlm": { - "vision_encoder_attr": "model.visual", - "language_model_attr": "model.language_model", - }, - }, - "renderer": None, - } - ) - - config = OrchestratorConfig.model_validate( - { - "model": { - "name": "Qwen/Qwen3-VL-4B-Instruct", - "vlm": { - "vision_encoder_attr": "model.visual", - "language_model_attr": "model.language_model", - }, - }, - } - ) - - assert config.renderer is not None - - def test_trainer_rejects_vlm_cp_with_ring(): config = { "model": { "cp": 2, "optimization_dtype": "bfloat16", "reduce_dtype": "bfloat16", - "vlm": { - "vision_encoder_attr": "model.visual", - "language_model_attr": "model.language_model", - }, + "vlm": {}, }, }