diff --git a/nemo_automodel/_transformers/auto_model.py b/nemo_automodel/_transformers/auto_model.py index 01ce00fa13..da7b1ffbab 100644 --- a/nemo_automodel/_transformers/auto_model.py +++ b/nemo_automodel/_transformers/auto_model.py @@ -27,6 +27,7 @@ import gc import inspect import logging +import os from contextlib import nullcontext from typing import TYPE_CHECKING, List, Optional, Union @@ -96,6 +97,7 @@ no_hf_meta_device, resolve_sdpa_method, ) +from nemo_automodel.components.models.common.tie_word_embeddings import reject_tie_word_embeddings_flip if not hasattr(_gen_utils, "NEED_SETUP_CACHE_CLASSES_MAPPING"): from transformers.cache_utils import StaticCache @@ -254,6 +256,42 @@ def _maybe_dequantize_fp8_for_peft(hf_native_quant_cfg, peft_config, pretrained_ return False +def _maybe_reject_tie_word_embeddings_flip(pretrained_model_name_or_path, hf_config, kwargs): + """Reject a from_pretrained request that flips tie_word_embeddings from the checkpoint. + + Re-reads the checkpoint's raw config (no user value-overrides) and compares its + controlling tie flag to the requested ``hf_config`` via + :func:`reject_tie_word_embeddings_flip`. Conservative by design: path-like sources + are normalized with :func:`os.fspath`, non-path sources are skipped, and it silently + returns if the raw config cannot be re-read, so it never blocks a load except on a + genuine flip. + + Args: + pretrained_model_name_or_path: The from_pretrained source (``str`` and + ``os.PathLike`` are checked; anything else is skipped). + hf_config: The resolved config with user overrides applied (the requested value). + kwargs: The from_pretrained kwargs (hub-locating keys are reused for the raw load). + """ + if isinstance(pretrained_model_name_or_path, os.PathLike): + pretrained_model_name_or_path = os.fspath(pretrained_model_name_or_path) + if not isinstance(pretrained_model_name_or_path, str): + # Non-path source (e.g. bytes fspath or preloaded object): nothing to re-read. + return + hub_kwargs = {k: kwargs[k] for k in _AUTO_CONFIG_HUB_KWARG_KEYS if k in kwargs} + try: + raw_config = AutoConfig.from_pretrained( + pretrained_model_name_or_path, + trust_remote_code=kwargs.get("trust_remote_code", resolve_trust_remote_code(pretrained_model_name_or_path)), + **hub_kwargs, + ) + except Exception: + # Cannot re-read the raw config (offline / custom loader); do not block the load. + return + architectures = getattr(hf_config, "architectures", None) or [] + model_class_name = architectures[0] if architectures else type(hf_config).__name__ + reject_tie_word_embeddings_flip(raw_config, hf_config, model_class_name) + + class _BaseNeMoAutoModelClass(_BaseAutoModelClass): """ Drop-in replacement for ``_BaseAutoModelClass`` that includes custom-kernels. @@ -710,6 +748,10 @@ def from_pretrained( raise is_hf_model = get_is_hf_model(hf_config, force_hf) + # Layer 2: reject loading a checkpoint with tie_word_embeddings flipped from the + # value it was saved with (the class-level TieSupport policy cannot catch this). + _maybe_reject_tie_word_embeddings_flip(pretrained_model_name_or_path, hf_config, kwargs) + sdpa_method = resolve_sdpa_method(sdpa_method, mesh.device_mesh, activation_checkpointing) return cls._build_model( diff --git a/nemo_automodel/_transformers/model_init.py b/nemo_automodel/_transformers/model_init.py index 2575d91adf..5993041035 100644 --- a/nemo_automodel/_transformers/model_init.py +++ b/nemo_automodel/_transformers/model_init.py @@ -1001,7 +1001,7 @@ def _tie_weights_nemo(model): # model is tied. Re-tying an untied model here would alias away the trained # ``lm_head.weight`` that ``from_pretrained`` just loaded (see #2941). config = getattr(model, "config", None) - if config is not None and not checkpoint_utils.get_controlling_tie_word_embeddings(config, type(model).__name__): + if config is not None and not checkpoint_utils.is_tied_word_embeddings(model): return def get_module_by_fqn(model, fqn): diff --git a/nemo_automodel/components/checkpoint/utils.py b/nemo_automodel/components/checkpoint/utils.py index 221b1c53b2..3a641cbd53 100644 --- a/nemo_automodel/components/checkpoint/utils.py +++ b/nemo_automodel/components/checkpoint/utils.py @@ -22,6 +22,8 @@ import torch.nn as nn from transformers.modeling_utils import _get_resolved_checkpoint_files, load_state_dict +from nemo_automodel.components.models.common.tie_word_embeddings import TieSupport, get_controlling_tie_word_embeddings + def get_rank_safe() -> int: """Return the current distributed rank, defaulting to 0 when not initialized.""" @@ -101,71 +103,11 @@ def resolve_trust_remote_code(pretrained_model_name_or_path): return not os.path.isdir(pretrained_model_name_or_path) and pretrained_model_name_or_path.startswith("nvidia/") -def get_controlling_tie_word_embeddings(config: object, model_class_name: str) -> bool: - """Resolve the ``tie_word_embeddings`` flag that actually controls lm_head tying. - - HF ties ``lm_head`` based on the *top-level* config flag, not a nested - ``text_config`` (verified by construction for Gemma4 and Mistral3 under - transformers 5.8.1: the top-level flag decides tying regardless of the nested - value). So prefer the top-level flag, and only fall back to ``text_config`` - for configs that don't expose a top-level ``tie_word_embeddings``. - - Omni "thinker" models are the exception: the full wrapper config - (``Qwen2_5OmniConfig`` / ``Qwen3OmniMoeConfig``) does not expose - ``tie_word_embeddings`` at the top level at all -- the controlling flag lives - on ``config.thinker_config`` -- so unwrap to it for those classes. - - Args: - config: The model's config (or anything exposing ``tie_word_embeddings`` - and optionally ``get_text_config``). - model_class_name: ``type(model).__name__`` of the owning model class. - - Returns: - bool: The controlling ``tie_word_embeddings`` value. - """ - # Omni "thinker" models: the controlling flag lives on the thinker config. - # A full wrapper config (e.g. ``Qwen2_5OmniConfig`` / ``Qwen3OmniMoeConfig``) - # does not expose ``tie_word_embeddings`` at the top level at all -- it nests - # under ``config.thinker_config`` -- so unwrap to it when present. When the - # thinker config itself is passed, ``thinker_config`` is absent and we read - # its own top-level flag. - omni_thinker_models = ( - "Qwen2_5OmniThinkerForConditionalGeneration", - "Qwen3OmniMoeThinkerForConditionalGeneration", - ) - if any(name in model_class_name for name in omni_thinker_models): - thinker_config = getattr(config, "thinker_config", config) - return bool(getattr(thinker_config, "tie_word_embeddings", False)) - - # Other composite models whose top-level config owns the lm_head tying - # decision; their nested ``text_config`` flag can disagree and must be ignored. - # Return the top-level flag (not a forced ``False``) so a constructor guard can - # still see and reject an unsupported ``top-level=True``. The checkpoint save - # path stays safe through the storage-based ``has_local_tied_lm_head()`` check, - # which only drops ``lm_head.weight`` when the tensors actually share storage. - composite_top_level_models = ( - "Mistral3FP8VLMForConditionalGeneration", - "Qwen3VLMoeForConditionalGeneration", - ) - if any(name in model_class_name for name in composite_top_level_models): - return bool(getattr(config, "tie_word_embeddings", False)) - - # General rule: the top-level config wins when it exposes the flag. - if hasattr(config, "tie_word_embeddings"): - return bool(config.tie_word_embeddings) - - # Fallback only for configs that do not expose a top-level tie flag. - text_config = getattr(config, "get_text_config", lambda: None)() - return bool(getattr(text_config, "tie_word_embeddings", False)) - - def is_tied_word_embeddings(model: nn.Module) -> bool: - """ - Check if the model's word embeddings are tied. + """Check whether the model's word embeddings are tied. - Delegates to :func:`get_controlling_tie_word_embeddings`, which follows HF's - top-level-first tying semantics (replacing the previous ``text_config``-first - resolution). + A one-direction :class:`TieSupport` policy is authoritative. Only ``BOTH`` + models and undeclared Hugging Face models need their config flag resolved. Args: model (nn.Module): The model to check. @@ -173,65 +115,19 @@ def is_tied_word_embeddings(model: nn.Module) -> bool: Returns: bool: True if the model's word embeddings are tied, False otherwise. """ + support = getattr(model, "tie_word_embeddings_support", None) + # Composite VLM configs can expose an outer tie flag that does not describe + # the text head (for example Mistral4). No current BOTH VLM has that exception: + # they all honor the outer flag, so a model-owned resolver is not needed yet. + if support is TieSupport.TIED_ONLY: + return True + if support is TieSupport.UNTIED_ONLY: + return False + config = getattr(model, "config", None) if config is None: return False - return get_controlling_tie_word_embeddings(config, type(model).__name__) - - -def reject_unsupported_tied_word_embeddings(config: object, model_class_name: str) -> None: - """Reject ``tie_word_embeddings=True`` for models whose HF default is untied. - - Separate-head architectures (HF default: distinct input/output embeddings) - don't build a shared ``lm_head``, so honoring ``tie_word_embeddings=True`` - would silently leave a randomly-initialized head or require materializing a - tied weight NeMo does not support. Reject it explicitly with a clear message - instead of pretending to support it. - - Uses :func:`get_controlling_tie_word_embeddings`, so composite VLM/omni configs - are read from the controlling top-level flag rather than a nested - ``text_config``. - - Args: - config: The model's config. - model_class_name: ``type(self).__name__`` of the constructing model. - - Raises: - NotImplementedError: if the controlling ``tie_word_embeddings`` flag is set. - """ - if get_controlling_tie_word_embeddings(config, model_class_name): - raise NotImplementedError( - f"{model_class_name} has separate input and output embeddings and does not " - f"support tie_word_embeddings=True. The Hugging Face default for this " - f"architecture is untied; set tie_word_embeddings=False." - ) - - -def reject_unsupported_untied_word_embeddings(config: object, model_class_name: str) -> None: - """Reject ``tie_word_embeddings=False`` for models whose HF default is tied. - - Tied-by-default architectures share ``lm_head`` with the input embedding and - ship checkpoints without a separate ``lm_head.weight``. Honoring - ``tie_word_embeddings=False`` would require materializing a distinct - ``lm_head`` NeMo does not build (and a tied checkpoint has no weights for it), - so reject it explicitly instead of running with a randomly-initialized head. - - The mirror of :func:`reject_unsupported_tied_word_embeddings`; both read the - controlling flag via :func:`get_controlling_tie_word_embeddings`. - - Args: - config: The model's config. - model_class_name: ``type(self).__name__`` of the constructing model. - - Raises: - NotImplementedError: if the controlling ``tie_word_embeddings`` flag evaluates to ``False``. - """ - if not get_controlling_tie_word_embeddings(config, model_class_name): - raise NotImplementedError( - f"{model_class_name} ties its input and output embeddings and does not " - f"support tie_word_embeddings=False. The Hugging Face default for this " - f"architecture is tied; set tie_word_embeddings=True." - ) + return get_controlling_tie_word_embeddings(config) def _normalize_param_name(name: str) -> str: diff --git a/nemo_automodel/components/distributed/pipelining/hf_utils.py b/nemo_automodel/components/distributed/pipelining/hf_utils.py index 29d278c87c..419023f3fc 100644 --- a/nemo_automodel/components/distributed/pipelining/hf_utils.py +++ b/nemo_automodel/components/distributed/pipelining/hf_utils.py @@ -752,7 +752,9 @@ def validate_hf_model_for_pipeline_support(model: torch.nn.Module) -> None: ) if weights_tied: issues.append( - "tie_word_embeddings=True is not supported for pipelining. Use separate input/output embeddings." + "Pipeline parallelism does not support tie_word_embeddings=True, and overriding " + "it to tie_word_embeddings=False is not supported either. Train this model with " + "another supported parallelism strategy (e.g., FSDP2) instead." ) if getattr(config, "is_encoder_decoder", False): issues.append("Encoder-Decoder models with cross-attention are not supported yet for pipeline parallelism.") diff --git a/nemo_automodel/components/models/bagel/model.py b/nemo_automodel/components/models/bagel/model.py index 83df497790..b8e3c41a6e 100644 --- a/nemo_automodel/components/models/bagel/model.py +++ b/nemo_automodel/components/models/bagel/model.py @@ -50,6 +50,10 @@ load_bagel_checkpoint_state_dict, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) logger = logging.getLogger(__name__) @@ -213,6 +217,9 @@ class BagelForUnifiedMultimodal(HFCheckpointingMixin, nn.Module): """ config_class = BagelConfig + # BAGEL's served checkpoints are untied; the tie flag lives on the nested + # text_config (aliased as llm_config), and the inner Qwen2 LM owns the head. + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY @dataclass(frozen=True) class ModelCapabilities: @@ -225,6 +232,10 @@ class ModelCapabilities: def __init__(self, config: BagelConfig) -> None: super().__init__() + # Also covers the build_bagel_from_hf_backbones registry-bypass path, which + # constructs this class directly. Reads the nested text_config tie flag via + # the resolver's get_text_config fallback (BagelConfig has no top-level flag). + reject_unsupported_tie_word_embeddings(type(self), config) _prepare_config_for_stage(config) self.config = config self.model = BagelModel(config) diff --git a/nemo_automodel/components/models/baichuan/model.py b/nemo_automodel/components/models/baichuan/model.py index 5a8f7a1230..e5379b9c0e 100644 --- a/nemo_automodel/components/models/baichuan/model.py +++ b/nemo_automodel/components/models/baichuan/model.py @@ -49,6 +49,10 @@ from nemo_automodel.components.models.baichuan.configuration import BaichuanConfig from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import compute_lm_head_logits from nemo_automodel.components.models.deprecation import warn_deprecated_model_class @@ -475,7 +479,9 @@ def custom_forward(*inputs): # Causal LM head # --------------------------------------------------------------------------- class BaichuanForCausalLM(HFCheckpointingMixin, BaichuanPreTrainedModel, GenerationMixin): - _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} + # lm_head is a weight-normalizing NormHead, so tying it to embed_tokens is + # semantically wrong; all shipped Baichuan checkpoints are untied. + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY @dataclass(frozen=True) class ModelCapabilities: @@ -488,6 +494,7 @@ class ModelCapabilities: def __init__(self, config: BaichuanConfig, **model_kwargs): warn_deprecated_model_class("BaichuanForCausalLM") + reject_unsupported_tie_word_embeddings(type(self), config) super().__init__(config) self.model = BaichuanModel(config) self.lm_head = NormHead(config.hidden_size, config.vocab_size, bias=False) diff --git a/nemo_automodel/components/models/common/tie_word_embeddings.py b/nemo_automodel/components/models/common/tie_word_embeddings.py new file mode 100644 index 0000000000..da1220b6e9 --- /dev/null +++ b/nemo_automodel/components/models/common/tie_word_embeddings.py @@ -0,0 +1,161 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Model-owned policy for input and output embedding ties.""" + +from enum import Enum + + +class TieSupport(Enum): + """Which ``tie_word_embeddings`` settings a model class supports. + + Declared as the ``tie_word_embeddings_support`` class attribute on every + registered model class and consulted by + :func:`reject_unsupported_tie_word_embeddings` at construction time to reject + a config whose tying the architecture cannot honor. + """ + + #: Both tied and untied heads are supported: the class ties in ``__init__`` / + #: ``tie_weights()`` when requested and otherwise runs a separate ``lm_head``. + BOTH = "both" + #: Only ``tie_word_embeddings=True`` is supported: the architecture ties its + #: input and output embeddings and ships checkpoints without a distinct + #: ``lm_head.weight``. + TIED_ONLY = "tied_only" + #: Only ``tie_word_embeddings=False`` is supported: the architecture has + #: separate input and output embeddings and never builds a shared ``lm_head``. + UNTIED_ONLY = "untied_only" + + +def get_controlling_tie_word_embeddings(config: object) -> bool: + """Resolve the ``tie_word_embeddings`` flag that actually controls lm_head tying. + + HF ties ``lm_head`` based on the *top-level* config flag, not a nested + ``text_config`` (verified by construction for Gemma4 and Mistral3 under + transformers 5.8.1: the top-level flag decides tying regardless of the nested + value). So prefer the top-level flag, and only fall back to ``text_config`` + for configs that don't expose a top-level ``tie_word_embeddings``. + + Full Omni wrapper configs do not expose ``tie_word_embeddings`` at the top + level; the controlling flag lives on ``config.thinker_config``. Therefore, + after checking the top-level flag, fall back to ``thinker_config`` and then + ``text_config``. + + Args: + config: The model's config (or anything exposing ``tie_word_embeddings`` + and optionally ``thinker_config`` or ``get_text_config``). + + Returns: + The controlling ``tie_word_embeddings`` value. + """ + # General rule: the top-level config wins when it exposes the flag. + if hasattr(config, "tie_word_embeddings"): + return bool(config.tie_word_embeddings) + + thinker_config = getattr(config, "thinker_config", None) + if thinker_config is not None: + return bool(getattr(thinker_config, "tie_word_embeddings", False)) + + # Final fallback for text-only configs without a top-level tie flag. + text_config = getattr(config, "get_text_config", lambda: None)() + return bool(getattr(text_config, "tie_word_embeddings", False)) + + +def reject_unsupported_tie_word_embeddings(model_cls: type, config: object) -> None: + """Reject a ``tie_word_embeddings`` setting the model class cannot honor. + + Reads the class's declared :class:`TieSupport` policy from + ``model_cls.tie_word_embeddings_support`` (defaulting to + :attr:`TieSupport.BOTH`) and the controlling ``tie_word_embeddings`` flag + from :func:`get_controlling_tie_word_embeddings`, then raises when the + requested tying falls outside the supported set: + + - ``UNTIED_ONLY`` classes are separate-head architectures (HF default: distinct + input/output embeddings) that never build a shared ``lm_head``; honoring + ``tie_word_embeddings=True`` would leave a randomly-initialized head. + - ``TIED_ONLY`` classes share ``lm_head`` with the input embedding and ship + checkpoints without a distinct ``lm_head.weight``; honoring + ``tie_word_embeddings=False`` would require materializing a head NeMo does + not build and the checkpoint has no weights for. + + Call at the top of ``__init__`` on the *original* top-level config, before + unwrapping to ``text_config`` / ``thinker_config`` or calling + ``super().__init__()``, so the correct controlling flag is inspected and + construction fails fast. + + Args: + model_cls: The constructing model class (``type(self)``). Its + ``tie_word_embeddings_support`` attribute declares the policy, and its + name is included in validation errors. + config: The model's original (top-level) config. + + Raises: + NotImplementedError: If tying is requested on a + :attr:`TieSupport.UNTIED_ONLY` class, or untying is requested on a + :attr:`TieSupport.TIED_ONLY` class. + """ + support = getattr(model_cls, "tie_word_embeddings_support", TieSupport.BOTH) + if support is TieSupport.BOTH: + return + + model_class_name = model_cls.__name__ + requested_tied = get_controlling_tie_word_embeddings(config) + + if support is TieSupport.UNTIED_ONLY and requested_tied: + raise NotImplementedError( + f"{model_class_name} has separate input and output embeddings and does not " + f"support tie_word_embeddings=True. The Hugging Face default for this " + f"architecture is untied; set tie_word_embeddings=False." + ) + if support is TieSupport.TIED_ONLY and not requested_tied: + raise NotImplementedError( + f"{model_class_name} ties its input and output embeddings and does not " + f"support tie_word_embeddings=False. The Hugging Face default for this " + f"architecture is tied; set tie_word_embeddings=True." + ) + + +def reject_tie_word_embeddings_flip(checkpoint_config: object, requested_config: object, model_class_name: str) -> None: + """Reject loading a checkpoint with ``tie_word_embeddings`` flipped from its own value. + + The class-level :class:`TieSupport` declaration cannot catch flipping the flag away + from a *specific* checkpoint's value (a ``BOTH`` class accepts either) -- that is a + ``(checkpoint, requested)`` property. NeMo AutoModel respects the checkpoint's tie + semantics, so a mismatch in either direction is rejected: + + - untied checkpoint requested tied -> would silently discard the trained ``lm_head``; + - tied checkpoint requested untied -> would leave a randomly-initialized ``lm_head`` + (the adapters' embed->head copy is gated on the flag). + + Only applies to ``from_pretrained`` (there is a loaded checkpoint to compare against); + ``from_config`` / scratch has no checkpoint, so the user's config is authoritative. + + Args: + checkpoint_config: The config parsed from the checkpoint, before user overrides. + requested_config: The config after user overrides are applied. + model_class_name: The resolved model class name to include in validation errors. + + Raises: + NotImplementedError: If the controlling ``tie_word_embeddings`` flag differs + between the checkpoint and the requested config. + """ + checkpoint_tied = get_controlling_tie_word_embeddings(checkpoint_config) + requested_tied = get_controlling_tie_word_embeddings(requested_config) + if checkpoint_tied != requested_tied: + raise NotImplementedError( + f"{model_class_name}: requested tie_word_embeddings={requested_tied} but the checkpoint " + f"declares tie_word_embeddings={checkpoint_tied}. NeMo AutoModel respects the checkpoint's " + f"tie semantics; flipping the flag is not supported (it would leave a randomly-initialized " + f"or discarded lm_head). Load the checkpoint with its own tie_word_embeddings value." + ) diff --git a/nemo_automodel/components/models/deepseek_v3/model.py b/nemo_automodel/components/models/deepseek_v3/model.py index fd1b4e76ed..d6da76259b 100644 --- a/nemo_automodel/components/models/deepseek_v3/model.py +++ b/nemo_automodel/components/models/deepseek_v3/model.py @@ -20,7 +20,6 @@ from transformers.modeling_outputs import CausalLMOutputWithPast from transformers.models.deepseek_v3.configuration_deepseek_v3 import DeepseekV3Config -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, get_rope_config, @@ -28,6 +27,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import compute_lm_head_logits, yield_fp32_model from nemo_automodel.components.models.deepseek_v3.layers import MLA from nemo_automodel.components.models.deepseek_v3.rope_utils import freqs_cis_from_position_ids, precompute_freqs_cis @@ -258,6 +261,7 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class DeepseekV3ForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY _keep_in_fp32_modules_strict = ["e_score_correction_bias"] @dataclass(frozen=True) @@ -300,7 +304,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() # The HF DeepSeek-V3 reference computes router scoring in fp32; routing is highly # precision-sensitive (small bf16 errors flip expert selection) and the gate is tiny, diff --git a/nemo_automodel/components/models/deepseek_v32/model.py b/nemo_automodel/components/models/deepseek_v32/model.py index 0f8f437c2f..a2b844c6fc 100644 --- a/nemo_automodel/components/models/deepseek_v32/model.py +++ b/nemo_automodel/components/models/deepseek_v32/model.py @@ -26,13 +26,16 @@ import torch.nn as nn from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, compute_lm_head_logits, get_rope_config, initialize_rms_norm_module, ) +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.deepseek_v3.model import ( Block, DeepseekV3ForCausalLM, @@ -167,6 +170,8 @@ class DeepseekV32ForCausalLM(DeepseekV3ForCausalLM): Subclasses V3 ForCausalLM, using DeepseekV32Model and DeepSeekV32StateDictAdapter. """ + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" @@ -209,7 +214,7 @@ def __init__( from nemo_automodel.components.models.common import initialize_linear_module self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() # Use V3.2 Model instead of V3 Model moe_overrides = kwargs.pop("moe_overrides", None) diff --git a/nemo_automodel/components/models/deepseek_v4/model.py b/nemo_automodel/components/models/deepseek_v4/model.py index d9553efbe9..848acafd91 100644 --- a/nemo_automodel/components/models/deepseek_v4/model.py +++ b/nemo_automodel/components/models/deepseek_v4/model.py @@ -49,13 +49,16 @@ import torch.nn as nn from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, initialize_linear_module, initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import ( _has_dtensor_params, cast_model_to_dtype, @@ -621,6 +624,7 @@ class DeepseekV4ForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): # ``DeepseekV4PreTrainedModel._keep_in_fp32_modules_strict`` (lines 890-900 # of modular_deepseek_v4.py) plus the existing ``e_score_correction_bias`` # entry that is specific to KAutomodel's shared Gate buffer. + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY _keep_in_fp32_modules_strict = [ "attn_hc.fn", "attn_hc.base", @@ -687,7 +691,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) mtp_loss_scaling_factor = kwargs.pop("mtp_loss_scaling_factor", 0.1) diff --git a/nemo_automodel/components/models/diffusion_gemma/model.py b/nemo_automodel/components/models/diffusion_gemma/model.py index 188061190d..9004eabc8c 100644 --- a/nemo_automodel/components/models/diffusion_gemma/model.py +++ b/nemo_automodel/components/models/diffusion_gemma/model.py @@ -54,6 +54,10 @@ from nemo_automodel._transformers.model_capabilities import ModelCapabilities from nemo_automodel.components.models.common import BackendConfig from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype from nemo_automodel.components.moe.config import MoEConfig from nemo_automodel.components.moe.fsdp_mixin import MoEFSDPSyncMixin @@ -335,7 +339,10 @@ class DiffusionGemmaForBlockDiffusion(HFCheckpointingMixin, MoEFSDPSyncMixin, Pr # modules afterwards (see llama/rope_utils.py). _keep_in_fp32_modules = ["rotary_emb"] _no_split_modules = ["DiffusionGemmaMoEDecoderLayer"] - _tied_weights_keys = ["lm_head.weight"] + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} + # lm_head is hard-tied to the shared embedding; the adapter drops it on export + # and rebuilds it from the embedding on load, so untying is unsupported. + tie_word_embeddings_support: TieSupport = TieSupport.TIED_ONLY @classmethod def get_capabilities(cls, config: "DiffusionGemmaConfig") -> "ModelCapabilities": @@ -372,6 +379,7 @@ def __init__( freeze_router: bool | None = None, **kwargs: Any, ): + reject_unsupported_tie_word_embeddings(type(self), config) # ``canvas_length`` is a declared field on the reference config (and round-trips), # so a YAML/from_pretrained override is written back onto it. The training-only # flags are NOT reference-config fields (it is a strict dataclass), so they live on @@ -406,9 +414,7 @@ def __init__( self.model = DiffusionGemmaBackbone(text_config, self.backend, moe_config=moe_config) self.lm_head = nn.Linear(text_config.hidden_size, text_config.vocab_size, bias=False) - # lm_head is tied to the shared embedding (HF resolves this via - # _tied_weights_keys + get_output_embeddings). - self.lm_head.weight = self.model.embed_tokens.weight + self.tie_weights() # Expose moe_config for the MoE parallelizer assertion path. self.moe_config = self.model.moe_config @@ -437,6 +443,10 @@ def set_input_embeddings(self, value: nn.Module) -> None: def get_output_embeddings(self) -> nn.Module: return self.lm_head + def tie_weights(self, *_args: object, **_kwargs: object) -> None: + """Tie ``lm_head`` to the shared diffusion text embedding.""" + self.lm_head.weight = self.model.embed_tokens.weight + def freeze_router_params(self) -> None: """Freeze the MoE router/gate (design v2 item 9). diff --git a/nemo_automodel/components/models/ernie4_5/model.py b/nemo_automodel/components/models/ernie4_5/model.py index 66fad5c135..3e9ccad111 100644 --- a/nemo_automodel/components/models/ernie4_5/model.py +++ b/nemo_automodel/components/models/ernie4_5/model.py @@ -35,6 +35,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.ernie4_5.rope_utils import Ernie4_5RotaryEmbedding, apply_rotary_pos_emb from nemo_automodel.components.models.ernie4_5.state_dict_adapter import ( Ernie4_5_MoeStateDictAdapter, @@ -411,6 +415,8 @@ def forward( class Ernie4_5ForCausalLM(HFCheckpointingMixin, nn.Module): """Dense ERNIE 4.5 causal language model.""" + # Both shipped ERNIE-4.5 checkpoints are tied; untied is not a validated path. + tie_word_embeddings_support: TieSupport = TieSupport.TIED_ONLY supports_gradient_checkpointing = True _skip_init_weights_on_load = True _nemo_tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} @@ -444,6 +450,7 @@ def __init__( ): super().__init__() self.config = config + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() self.model = Ernie4_5Model(config, self.backend) self.vocab_size = config.vocab_size @@ -515,6 +522,8 @@ def forward( class Ernie4_5_MoeForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): """ERNIE 4.5 MoE causal language model with AutoModel EP support.""" + # Both shipped ERNIE-4.5 checkpoints are tied; untied is not a validated path. + tie_word_embeddings_support: TieSupport = TieSupport.TIED_ONLY supports_gradient_checkpointing = True _skip_init_weights_on_load = True _nemo_tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} @@ -569,6 +578,7 @@ def __init__( ): super().__init__() self.config = config + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) self.model = Ernie4_5_MoeModel( diff --git a/nemo_automodel/components/models/gemma4_drafter/model.py b/nemo_automodel/components/models/gemma4_drafter/model.py index 26b56cfe01..b2958b77d4 100644 --- a/nemo_automodel/components/models/gemma4_drafter/model.py +++ b/nemo_automodel/components/models/gemma4_drafter/model.py @@ -23,6 +23,10 @@ from dataclasses import dataclass from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.shared.import_utils import UnavailableError, UnavailableMeta @@ -55,6 +59,19 @@ class Gemma4DrafterForCausalLM(HFCheckpointingMixin, HFGemma4AssistantForCausalL under a stable native class name. """ + # Only tied Gemma4 assistant checkpoints ship; untying is unsupported. + tie_word_embeddings_support: TieSupport = TieSupport.TIED_ONLY + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} + + def __init__(self, config: Gemma4AssistantConfig, *args, **kwargs): + reject_unsupported_tie_word_embeddings(type(self), config) + super().__init__(config, *args, **kwargs) + self.tie_weights() + + def tie_weights(self, *_args: object, **_kwargs: object) -> None: + """Tie ``lm_head`` to the drafter token embedding.""" + self.lm_head.weight = self.model.embed_tokens.weight + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" diff --git a/nemo_automodel/components/models/gemma4_moe/model.py b/nemo_automodel/components/models/gemma4_moe/model.py index e59f0f4837..99c8f7ed65 100644 --- a/nemo_automodel/components/models/gemma4_moe/model.py +++ b/nemo_automodel/components/models/gemma4_moe/model.py @@ -78,9 +78,12 @@ def _make_missing(name: str): CausalLMOutputWithPast = _make_missing("CausalLMOutputWithPast") from nemo_automodel._transformers.model_capabilities import ModelCapabilities -from nemo_automodel.components.checkpoint.utils import reject_unsupported_untied_word_embeddings from nemo_automodel.components.models.common import BackendConfig, compute_lm_head_logits from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype from nemo_automodel.components.moe.fsdp_mixin import MoEFSDPSyncMixin from nemo_automodel.components.moe.layers import MoE, MoEConfig @@ -811,6 +814,7 @@ def norm(self): # Top-level conditional-generation model # --------------------------------------------------------------------------- class Gemma4ForConditionalGeneration(HFCheckpointingMixin, HFGemma4ForConditionalGeneration, MoEFSDPSyncMixin): + tie_word_embeddings_support: TieSupport = TieSupport.TIED_ONLY supports_gradient_checkpointing = True # RoPE inv_freq must stay fp32: initialize_weights casts the model to bf16 and # nn.Module.to rounds floating buffers; cast_model_to_dtype restores keep-fp32 @@ -956,7 +960,7 @@ def __init__( raise UnavailableError("transformers.models.gemma4 is not available.") # Gemma4 is tied by default; untying would need a materialized separate # lm_head NeMo doesn't build, so reject tie_word_embeddings=False up front. - reject_unsupported_untied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) backend = backend or BackendConfig() # Merge text_config overrides (e.g. from YAML) into the proper config diff --git a/nemo_automodel/components/models/glm4_moe/model.py b/nemo_automodel/components/models/glm4_moe/model.py index 00fdb1dfa4..121489d1e6 100644 --- a/nemo_automodel/components/models/glm4_moe/model.py +++ b/nemo_automodel/components/models/glm4_moe/model.py @@ -20,7 +20,6 @@ from transformers.modeling_outputs import CausalLMOutputWithPast from transformers.models.glm4_moe.configuration_glm4_moe import Glm4MoeConfig -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, get_rope_config, @@ -28,6 +27,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.deprecation import warn_deprecated_model_class from nemo_automodel.components.models.glm4_moe.layers import Glm4MoeAttention @@ -234,6 +237,7 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class Glm4MoeForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY _keep_in_fp32_modules_strict = ["e_score_correction_bias"] @dataclass(frozen=True) @@ -275,7 +279,7 @@ def __init__( warn_deprecated_model_class("Glm4MoeForCausalLM") super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) self.model = Glm4MoeModel( diff --git a/nemo_automodel/components/models/glm4_moe_lite/model.py b/nemo_automodel/components/models/glm4_moe_lite/model.py index c189d098d4..9c659910c5 100644 --- a/nemo_automodel/components/models/glm4_moe_lite/model.py +++ b/nemo_automodel/components/models/glm4_moe_lite/model.py @@ -19,8 +19,11 @@ import torch.nn as nn from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import ( BackendConfig, cast_model_to_dtype, @@ -233,6 +236,7 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class Glm4MoeLiteForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY _keep_in_fp32_modules_strict = ["e_score_correction_bias"] @dataclass(frozen=True) @@ -277,7 +281,7 @@ def __init__( warn_deprecated_model_class("Glm4MoeLiteForCausalLM") super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) self.model = Glm4MoeLiteModel( diff --git a/nemo_automodel/components/models/glm_moe_dsa/model.py b/nemo_automodel/components/models/glm_moe_dsa/model.py index 573d869562..5b390a72f4 100644 --- a/nemo_automodel/components/models/glm_moe_dsa/model.py +++ b/nemo_automodel/components/models/glm_moe_dsa/model.py @@ -20,7 +20,6 @@ from transformers.modeling_outputs import CausalLMOutputWithPast from transformers.models.glm_moe_dsa.configuration_glm_moe_dsa import GlmMoeDsaConfig -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, compute_lm_head_logits, @@ -28,6 +27,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.deepseek_v3.rope_utils import ( freqs_cis_from_position_ids, precompute_freqs_cis, @@ -257,6 +260,8 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class GlmMoeDsaForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" @@ -296,7 +301,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) self.model = GlmMoeDsaModel( diff --git a/nemo_automodel/components/models/gpt_oss/model.py b/nemo_automodel/components/models/gpt_oss/model.py index 7289df64de..9beecd6548 100644 --- a/nemo_automodel/components/models/gpt_oss/model.py +++ b/nemo_automodel/components/models/gpt_oss/model.py @@ -21,7 +21,6 @@ from transformers.modeling_outputs import CausalLMOutputWithPast from transformers.models.gpt_oss.configuration_gpt_oss import GptOssConfig -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, get_rope_config, @@ -29,6 +28,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.gpt_oss.layers import GptOssAttention from nemo_automodel.components.models.gpt_oss.rope_utils import RotaryEmbedding, position_ids_to_freqs_cis @@ -216,6 +219,8 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class GptOssForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" @@ -254,7 +259,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig(attn="flex") moe_overrides = kwargs.pop("moe_overrides", None) self.model = GptOssModel(config, backend=self.backend, moe_config=moe_config, moe_overrides=moe_overrides) diff --git a/nemo_automodel/components/models/hy_mt2/model.py b/nemo_automodel/components/models/hy_mt2/model.py index c754bd897e..0df30240e7 100644 --- a/nemo_automodel/components/models/hy_mt2/model.py +++ b/nemo_automodel/components/models/hy_mt2/model.py @@ -43,7 +43,6 @@ import torch.nn as nn from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, get_rope_config, @@ -51,6 +50,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.gpt_oss.rope_utils import RotaryEmbedding, position_ids_to_freqs_cis from nemo_automodel.components.models.hy_mt2.layers import HyMT2Attention @@ -280,6 +283,8 @@ class HyMT2ForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): ``from_pretrained`` / ``save_pretrained`` over the HF safetensors layout. """ + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + _keep_in_fp32_modules_strict = ["mlp.gate.e_score_correction_bias"] @dataclass(frozen=True) @@ -326,7 +331,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) self.model = HyMT2Model(config, backend=self.backend, moe_config=moe_config, moe_overrides=moe_overrides) diff --git a/nemo_automodel/components/models/hy_v3/model.py b/nemo_automodel/components/models/hy_v3/model.py index 09d49e1860..0a8b3b9254 100644 --- a/nemo_automodel/components/models/hy_v3/model.py +++ b/nemo_automodel/components/models/hy_v3/model.py @@ -30,7 +30,6 @@ import torch.nn as nn from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, get_rope_config, @@ -38,6 +37,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.gpt_oss.rope_utils import RotaryEmbedding, position_ids_to_freqs_cis from nemo_automodel.components.models.hy_v3.layers import HYV3Attention @@ -221,6 +224,7 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class HYV3ForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY _keep_in_fp32_modules_strict = ["mlp.gate.e_score_correction_bias"] @dataclass(frozen=True) @@ -263,7 +267,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) self.model = HYV3Model(config, backend=self.backend, moe_config=moe_config, moe_overrides=moe_overrides) diff --git a/nemo_automodel/components/models/kimi_k25_vl/model.py b/nemo_automodel/components/models/kimi_k25_vl/model.py index 5fd5d95dcf..fd51e0295d 100644 --- a/nemo_automodel/components/models/kimi_k25_vl/model.py +++ b/nemo_automodel/components/models/kimi_k25_vl/model.py @@ -145,8 +145,11 @@ def to_dict(self) -> Dict[str, Any]: return output -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import BackendConfig, compute_lm_head_logits, initialize_linear_module +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.deepseek_v3.model import DeepseekV3Model from nemo_automodel.components.models.deepseek_v3.rope_utils import freqs_cis_from_position_ids from nemo_automodel.components.models.kimi_k25_vl.state_dict_adapter import KimiK25VLStateDictAdapter @@ -883,6 +886,8 @@ def forward( class KimiK25VLForConditionalGeneration(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): """KimiK25VL model with backend-aware DeepseekV3 language model.""" + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + # RoPE freqs/inv_freq must stay fp32: from_pretrained casts the model to bf16 and # nn.Module.to rounds floating buffers; routing through cast_model_to_dtype restores # these keep-fp32 buffers afterwards (see llama/rope_utils.py). @@ -952,7 +957,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: str, *model_args, **kwar def __init__(self, config, moe_config: MoEConfig | None = None, backend: BackendConfig | None = None, **kwargs): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() self.model = KimiK25VLModel(config, moe_config=moe_config, backend=self.backend) diff --git a/nemo_automodel/components/models/kimivl/model.py b/nemo_automodel/components/models/kimivl/model.py index e028d1d54b..1b151e7731 100644 --- a/nemo_automodel/components/models/kimivl/model.py +++ b/nemo_automodel/components/models/kimivl/model.py @@ -108,8 +108,11 @@ def to_dict(self) -> Dict[str, Any]: return output -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import BackendConfig, compute_lm_head_logits, initialize_linear_module +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.deepseek_v3.model import DeepseekV3Model from nemo_automodel.components.models.deepseek_v3.rope_utils import freqs_cis_from_position_ids from nemo_automodel.components.models.deepseek_v3.state_dict_adapter import DeepSeekV3StateDictAdapter @@ -634,6 +637,8 @@ def forward( class KimiVLForConditionalGeneration(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): """KimiVL model with backend-aware DeepseekV3 language model.""" + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + # forward() pulls per-microbatch pixel_values from _vlm_pixel_values_chunks; # patch_hf_model_for_pp must not replace it under PP. _pp_keep_self_forward: bool = True @@ -660,7 +665,7 @@ def __init__(self, config, moe_config: MoEConfig | None = None, backend: Backend warn_deprecated_model_class("KimiVLForConditionalGeneration") super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() self.model = KimiVLModel(config, moe_config=moe_config, backend=self.backend) diff --git a/nemo_automodel/components/models/ling_v2/model.py b/nemo_automodel/components/models/ling_v2/model.py index 862acc9677..44f334f7ce 100644 --- a/nemo_automodel/components/models/ling_v2/model.py +++ b/nemo_automodel/components/models/ling_v2/model.py @@ -42,13 +42,16 @@ from transformers.modeling_outputs import CausalLMOutputWithPast from nemo_automodel._transformers.model_capabilities import ModelCapabilities -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, initialize_linear_module, initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.gpt_oss.rope_utils import RotaryEmbedding, position_ids_to_freqs_cis from nemo_automodel.components.models.ling_v2.config import BailingMoeV2Config @@ -277,6 +280,8 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class BailingMoeV2ForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): """Causal-LM head wrapping ``BailingMoeV2Model``.""" + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + # ``e_score_correction_bias`` must stay in fp32 even when the rest of the # model is bf16; tiny quantization errors in the bias change routing. _keep_in_fp32_modules_strict = ["e_score_correction_bias"] @@ -343,7 +348,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) self.model = BailingMoeV2Model( diff --git a/nemo_automodel/components/models/llama/model.py b/nemo_automodel/components/models/llama/model.py index d9bd5f3d12..d3d28a0e12 100644 --- a/nemo_automodel/components/models/llama/model.py +++ b/nemo_automodel/components/models/llama/model.py @@ -51,6 +51,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.deprecation import warn_deprecated_model_class from nemo_automodel.components.models.llama.rope_utils import ( LlamaRotaryEmbedding, @@ -393,6 +397,7 @@ def forward( class LlamaForCausalLM(HFCheckpointingMixin, LlamaPreTrainedModel): """Llama model with causal language modeling head.""" + tie_word_embeddings_support: TieSupport = TieSupport.BOTH _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} _tp_plan = {"lm_head": "colwise_rep"} _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} @@ -420,6 +425,7 @@ def __init__( config: LlamaConfig, backend: Optional[BackendConfig] = None, ): + reject_unsupported_tie_word_embeddings(type(self), config) warn_deprecated_model_class("LlamaForCausalLM") super().__init__(config) self.config = config diff --git a/nemo_automodel/components/models/llama/state_dict_adapter.py b/nemo_automodel/components/models/llama/state_dict_adapter.py index 9e7d7eb674..c2ed45c46a 100644 --- a/nemo_automodel/components/models/llama/state_dict_adapter.py +++ b/nemo_automodel/components/models/llama/state_dict_adapter.py @@ -56,7 +56,8 @@ def from_hf(self, hf_state_dict: dict[str, Any], **kwargs) -> dict[str, Any]: # HF keys match model keys directly. # Only need to handle tied lm_head weights. custom_state_dict = dict(hf_state_dict) - if getattr(self.config, "tie_word_embeddings", True): + # Default False to match __init__/tie_weights (config always carries the flag). + if getattr(self.config, "tie_word_embeddings", False): embed_key = "model.embed_tokens.weight" lm_head_key = "lm_head.weight" if lm_head_key not in custom_state_dict and embed_key in custom_state_dict: diff --git a/nemo_automodel/components/models/llava_onevision/model.py b/nemo_automodel/components/models/llava_onevision/model.py index fb483d1a68..e2b238f659 100644 --- a/nemo_automodel/components/models/llava_onevision/model.py +++ b/nemo_automodel/components/models/llava_onevision/model.py @@ -34,8 +34,11 @@ from transformers.configuration_utils import PretrainedConfig from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import compute_lm_head_logits from nemo_automodel.components.models.llava_onevision.rice_vit import RiceTransformer @@ -284,6 +287,8 @@ def forward( class LLaVAOneVision1_5_ForConditionalGeneration(HFCheckpointingMixin, nn.Module): """LLaVA-OneVision-1.5 for conditional generation (Rice ViT + Qwen3 text).""" + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + config_class = Llavaonevision1_5Config @dataclass(frozen=True) @@ -314,7 +319,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) if attn_implementation is None: attn_implementation = getattr(config, "_attn_implementation", None) or "eager" self.model = LLaVAOneVision1_5_Model(config, attn_implementation=attn_implementation) diff --git a/nemo_automodel/components/models/mimo_v2_flash/model.py b/nemo_automodel/components/models/mimo_v2_flash/model.py index 6ab5581cde..56cfe64b63 100644 --- a/nemo_automodel/components/models/mimo_v2_flash/model.py +++ b/nemo_automodel/components/models/mimo_v2_flash/model.py @@ -23,12 +23,15 @@ from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, initialize_linear_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import ( _has_dtensor_params, cast_model_to_dtype, @@ -585,6 +588,8 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class MiMoV2FlashForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): """Causal LM wrapper for MiMo-V2-Flash with Automodel checkpoint adapters.""" + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + # "rotary_emb" (matches self.rotary_emb + self.swa_rotary_emb) pins their inv_freq # buffers in fp32: cast_model_to_dtype's bf16 cast would otherwise round inv_freq and # degrade RoPE precision vs HF (see llama/rope_utils.py). @@ -630,7 +635,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) self.model = MiMoV2FlashModel( diff --git a/nemo_automodel/components/models/minimax_m2/model.py b/nemo_automodel/components/models/minimax_m2/model.py index 2b48ba5e8c..27b9013fd8 100644 --- a/nemo_automodel/components/models/minimax_m2/model.py +++ b/nemo_automodel/components/models/minimax_m2/model.py @@ -19,7 +19,6 @@ import torch.nn as nn from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, get_rope_config, @@ -27,6 +26,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.gpt_oss.rope_utils import RotaryEmbedding, position_ids_to_freqs_cis from nemo_automodel.components.models.minimax_m2.layers import MiniMaxM2Attention @@ -225,6 +228,7 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class MiniMaxM2ForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY _keep_in_fp32_modules_strict = ["mlp.gate.e_score_correction_bias"] @dataclass(frozen=True) @@ -273,7 +277,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) self.model = MiniMaxM2Model( diff --git a/nemo_automodel/components/models/minimax_m3_vl/model.py b/nemo_automodel/components/models/minimax_m3_vl/model.py index 1ff4610139..a190aef76a 100644 --- a/nemo_automodel/components/models/minimax_m3_vl/model.py +++ b/nemo_automodel/components/models/minimax_m3_vl/model.py @@ -26,13 +26,16 @@ import torch import torch.nn as nn -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, get_rope_config, initialize_linear_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype from nemo_automodel.components.models.gpt_oss.rope_utils import RotaryEmbedding, position_ids_to_freqs_cis from nemo_automodel.components.models.minimax_m3_vl.config import MiniMaxM3VLConfig, MiniMaxM3VLTextConfig @@ -253,6 +256,8 @@ def mtp_logits( class MiniMaxM3SparseForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): """Standalone M3 text backbone for causal LM (Stage 1 parity target).""" + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + _keep_in_fp32_modules_strict = ["mlp.gate.e_score_correction_bias"] # The state-dict adapter loads every tensor from the checkpoint, so skip HF @@ -279,7 +284,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() self.model = MiniMaxM3TextModel(config, backend=self.backend, moe_config=moe_config) self.lm_head = initialize_linear_module(self.backend.linear, config.hidden_size, config.vocab_size, bias=False) @@ -368,6 +373,8 @@ class MiniMaxM3SparseForConditionalGeneration(HFCheckpointingMixin, nn.Module, M positions, then run through the (sparse/dense MoE) language model + lm_head. """ + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + # Pipeline-parallel routing: keep this VLM's own forward (which splices vision # features) instead of letting patch_hf_model_for_pp swap in the generic # CausalLM forward (which would drop pixel_values). MTP per-depth outputs are @@ -425,7 +432,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) text_config = config.text_config self.backend = backend or BackendConfig() self.model = MiniMaxM3TextModel(text_config, backend=self.backend, moe_config=moe_config) diff --git a/nemo_automodel/components/models/mistral3/model.py b/nemo_automodel/components/models/mistral3/model.py index a7c5b4cfd0..5f0a30ca4f 100644 --- a/nemo_automodel/components/models/mistral3/model.py +++ b/nemo_automodel/components/models/mistral3/model.py @@ -38,6 +38,10 @@ from transformers.utils import TransformersKwargs, can_return_tuple, logging from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import compute_lm_head_logits logger = logging.get_logger(__name__) @@ -505,6 +509,9 @@ def forward( class Ministral3ForCausalLM(HFCheckpointingMixin, Ministral3PreTrainedModel, GenerationMixin): + # No checkpoint served through this arch is tied (config default is untied); + # the 3B's text_config tie flag reaches the VLM class, not this one. + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} _tp_plan = {"lm_head": "colwise_rep"} _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} @@ -519,6 +526,7 @@ class ModelCapabilities: supports_ep: bool = False def __init__(self, config: Ministral3Config): + reject_unsupported_tie_word_embeddings(type(self), config) super().__init__(config) self.model = Ministral3Model(config) self.vocab_size = config.vocab_size diff --git a/nemo_automodel/components/models/mistral3_vlm/model.py b/nemo_automodel/components/models/mistral3_vlm/model.py index 142675058b..7e5f4a1542 100644 --- a/nemo_automodel/components/models/mistral3_vlm/model.py +++ b/nemo_automodel/components/models/mistral3_vlm/model.py @@ -43,7 +43,10 @@ Mistral3ForConditionalGeneration as _HFMistral3ForConditionalGeneration, ) -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import compute_lm_head_logits from nemo_automodel.components.models.mistral3_vlm.state_dict_adapter import ( Mistral3FP8StateDictAdapter, @@ -113,6 +116,13 @@ class Mistral3FP8VLMForConditionalGeneration(_HFMistral3ForConditionalGeneration Mistral3 VLM checkpoint (e.g. dawn-ridge-128B). """ + # This class serves both tied checkpoints (Ministral-3, whose lm_head is not + # serialized) and untied checkpoints (Mistral-Medium-3.5-128B, Devstral-24B, + # tie_word_embeddings=False). Per-checkpoint tie semantics are enforced by + # the from_pretrained flip guard, not at construction. + tie_word_embeddings_support: TieSupport = TieSupport.BOTH + _tied_weights_keys = {"lm_head.weight": "model.language_model.embed_tokens.weight"} + # See checkpointing.py:initialize_model_weights — gate on this attribute # to skip HF's ``initialize_weights()``. The upcoming adapter load will # populate every tensor, and skipping avoids a stage-divergent DTensor @@ -131,9 +141,7 @@ class ModelCapabilities: supports_ep: bool = False def __init__(self, config: PretrainedConfig): - # The supported Mistral3 checkpoint (mistralai/Mistral-Medium-3.5-128B) is - # untied (tie_word_embeddings=False), so reject tie_word_embeddings=True. - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) # HF's Mistral3ForConditionalGeneration.__init__ consults # ``config.quantization_config`` and swaps nn.Linear → FP8Linear for # every language_model Linear. FP8Linear registers a 0-d @@ -156,6 +164,7 @@ def __init__(self, config: PretrainedConfig): except AttributeError: pass super().__init__(config) + self.tie_weights() self.state_dict_adapter = Mistral3FP8StateDictAdapter.for_vlm_full(config) # Lazy non-persistent buffer reinit. HF's Ministral3RotaryEmbedding / @@ -178,6 +187,11 @@ def __init__(self, config: PretrainedConfig): sub._mistral3_fp8_rotary_reinit_done = False sub.register_forward_pre_hook(_rotary_reinit_self_hook, with_kwargs=True, prepend=True) + def tie_weights(self, *_args: object, **_kwargs: object) -> None: + """Tie ``lm_head`` to the active text embedding when requested.""" + if getattr(getattr(self, "config", None), "tie_word_embeddings", False): + self.lm_head.weight = self.model.language_model.embed_tokens.weight + def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/nemo_automodel/components/models/mistral4/model.py b/nemo_automodel/components/models/mistral4/model.py index bffa30907c..14d4cd553e 100644 --- a/nemo_automodel/components/models/mistral4/model.py +++ b/nemo_automodel/components/models/mistral4/model.py @@ -19,7 +19,6 @@ import torch.nn as nn from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, compute_lm_head_logits, @@ -28,6 +27,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.deepseek_v3.layers import MLA from nemo_automodel.components.models.deepseek_v3.model import Block from nemo_automodel.components.models.deepseek_v3.rope_utils import ( @@ -305,6 +308,8 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class Mistral4ForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" @@ -348,7 +353,7 @@ def __init__( super().__init__() # Reject an unsupported tied request on the controlling top-level flag # before unwrapping to text_config below. - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) # Extract text_config if this is a multimodal wrapper config config = getattr(config, "text_config", config) self.config = config @@ -675,6 +680,10 @@ class Mistral3ForConditionalGeneration(HFCheckpointingMixin, nn.Module, MoEFSDPS (not HF PreTrainedModel) to avoid FSDP conflicts. """ + # Head lives in the Mistral4 text backbone (separate lm_head, no tie + # mechanism); the controlling flag is on the nested text_config. + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" @@ -720,6 +729,9 @@ def __init__( **kwargs, ): super().__init__() + # Read the nested text_config flag: the composite Mistral3Config top-level + # defaults to tie=True and does not reflect the untied Mistral4 backbone. + reject_unsupported_tie_word_embeddings(type(self), config.text_config) backend = backend or BackendConfig() num_hidden_layers = kwargs.pop("num_hidden_layers", None) if num_hidden_layers is not None: diff --git a/nemo_automodel/components/models/nemotron_omni/model.py b/nemo_automodel/components/models/nemotron_omni/model.py index 3aeac5ee30..c493088908 100644 --- a/nemo_automodel/components/models/nemotron_omni/model.py +++ b/nemo_automodel/components/models/nemotron_omni/model.py @@ -34,9 +34,12 @@ from transformers.configuration_utils import PretrainedConfig from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import BackendConfig from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype from nemo_automodel.components.models.nemotron_v3.model import ( NemotronHForCausalLM as NemotronV3ForCausalLM, @@ -241,6 +244,8 @@ class NemotronOmniForConditionalGeneration(HFCheckpointingMixin, nn.Module, MoEF has custom DTensor parallelism for the Mamba+Attention hybrid MoE architecture. """ + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" @@ -304,7 +309,7 @@ def __init__( """ super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() # --------------------------------------------------------------- diff --git a/nemo_automodel/components/models/nemotron_parse/model.py b/nemo_automodel/components/models/nemotron_parse/model.py index 6a06015677..13768be659 100644 --- a/nemo_automodel/components/models/nemotron_parse/model.py +++ b/nemo_automodel/components/models/nemotron_parse/model.py @@ -38,8 +38,11 @@ MBartScaledWordEmbedding, ) -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import compute_lm_head_logits # ----------------------------------------------------------------------------- @@ -435,6 +438,8 @@ def _init_weights(self, module): class NemotronParseForConditionalGeneration(HFCheckpointingMixin, NemotronParsePreTrainedModel, GenerationMixin): """NemotronParse model for conditional generation tasks.""" + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" @@ -445,7 +450,7 @@ class ModelCapabilities: supports_ep: bool = False def __init__(self, config: NemotronParseConfig, loss_fn=None, **kwargs): - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) super().__init__(config) self.loss_fn = loss_fn diff --git a/nemo_automodel/components/models/nemotron_v3/model.py b/nemo_automodel/components/models/nemotron_v3/model.py index 94c078d3fb..025992533b 100644 --- a/nemo_automodel/components/models/nemotron_v3/model.py +++ b/nemo_automodel/components/models/nemotron_v3/model.py @@ -22,13 +22,16 @@ from transformers.generation import GenerationConfig, GenerationMixin from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import ( BackendConfig, HFCheckpointingMixin, initialize_linear_module, initialize_rms_norm_module, ) +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.nemotron_v3.layers import NemotronV3Block from nemo_automodel.components.models.nemotron_v3.mtp import ( @@ -283,6 +286,8 @@ class NemotronHForCausalLM(HFCheckpointingMixin, GenerationMixin, nn.Module, MoE per-step KV caching for attention layers and recurrent state caching for Mamba2 layers. """ + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + # Hybrid Mamba2/Attention uses NemotronHybridCache, not DynamicCache. _is_stateful: bool = True main_input_name: str = "input_ids" @@ -371,7 +376,7 @@ def __init__( """ super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() # Base model diff --git a/nemo_automodel/components/models/qwen2/model.py b/nemo_automodel/components/models/qwen2/model.py index ada824cc92..1276eca8cc 100644 --- a/nemo_automodel/components/models/qwen2/model.py +++ b/nemo_automodel/components/models/qwen2/model.py @@ -49,6 +49,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.deprecation import warn_deprecated_model_class # Use shared rope_utils (same implementation as Llama, supports both config formats) @@ -374,6 +378,7 @@ class Qwen2ForCausalLM(HFCheckpointingMixin, Qwen2PreTrainedModel): Uses separate q/k/v and gate/up projections -- HuggingFace layout. """ + tie_word_embeddings_support: TieSupport = TieSupport.BOTH _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} _tp_plan = {"lm_head": "colwise_rep"} _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} @@ -392,6 +397,7 @@ def __init__( config: Qwen2Config, backend: Optional[BackendConfig] = None, ): + reject_unsupported_tie_word_embeddings(type(self), config) warn_deprecated_model_class("Qwen2ForCausalLM") super().__init__(config) self.backend = backend or BackendConfig() @@ -431,6 +437,13 @@ def get_output_embeddings(self): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings + def tie_weights(self, *_args: object, **_kwargs: object) -> None: + # Transformers v5 does not reliably tie this custom model from the + # dict-shaped _tied_weights_keys alone; honor the config flag explicitly + # (mirrors LlamaForCausalLM). + if getattr(self.config, "tie_word_embeddings", False): + self.lm_head.weight = self.model.embed_tokens.weight + @can_return_tuple def forward( self, diff --git a/nemo_automodel/components/models/qwen2_5_omni/model.py b/nemo_automodel/components/models/qwen2_5_omni/model.py index 78cb5ef5e8..1e0a2b8303 100644 --- a/nemo_automodel/components/models/qwen2_5_omni/model.py +++ b/nemo_automodel/components/models/qwen2_5_omni/model.py @@ -44,9 +44,12 @@ Qwen2_5OmniThinkerForConditionalGeneration as HFQwen2_5OmniThinkerForConditionalGeneration, ) -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import BackendConfig, compute_lm_head_logits from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.qwen2_5_omni.state_dict_adapter import Qwen2_5OmniStateDictAdapter from nemo_automodel.shared.utils import dtype_from_str as get_dtype @@ -65,6 +68,8 @@ class Qwen2_5OmniThinkerForConditionalGeneration( ): """Qwen2.5-Omni Thinker (audio + image + video + text → text).""" + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" @@ -103,7 +108,7 @@ def __init__( ): # Check the controlling top-level flag on the original config before # resolving to thinker_config and building the HF parent. - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) thinker_config = _resolve_thinker_config(config) super().__init__(thinker_config) diff --git a/nemo_automodel/components/models/qwen3_5/model.py b/nemo_automodel/components/models/qwen3_5/model.py index 9d0736dcdf..80cc531ded 100644 --- a/nemo_automodel/components/models/qwen3_5/model.py +++ b/nemo_automodel/components/models/qwen3_5/model.py @@ -42,6 +42,10 @@ from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin from nemo_automodel.components.models.common.mtp import MTPConfig, MTPModule, roll_tensor from nemo_automodel.components.models.common.packing import is_indexed_packed_mask +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype from nemo_automodel.components.models.qwen3_5_moe.cp_linear_attn import CPAwareGatedDeltaNet from nemo_automodel.components.models.qwen3_next.layers import Qwen3NextRMSNorm @@ -589,6 +593,9 @@ def forward( class Qwen3_5ForCausalLM(HFCheckpointingMixin, nn.Module): """Qwen3.5 dense causal LM with optional Megatron-style MTP head.""" + tie_word_embeddings_support: TieSupport = TieSupport.BOTH + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" @@ -626,6 +633,7 @@ def __init__( num_nextn_predict_layers: int | None = None, **kwargs: Any, ) -> None: + reject_unsupported_tie_word_embeddings(type(self), config) super().__init__() del kwargs self.config = config @@ -667,8 +675,9 @@ def get_output_embeddings(self) -> nn.Module: def set_output_embeddings(self, new_embeddings: nn.Module) -> None: self.lm_head = new_embeddings - def tie_weights(self) -> None: - self.lm_head.weight = self.model.embed_tokens.weight + def tie_weights(self, *_args: object, **_kwargs: object) -> None: + if getattr(self.config, "tie_word_embeddings", False): + self.lm_head.weight = self.model.embed_tokens.weight def forward( self, @@ -794,6 +803,9 @@ class Qwen3_5ForConditionalGeneration(HFCheckpointingMixin, HFQwen3_5ForConditio # patch_hf_model_for_pp must not replace it under PP. _pp_keep_self_forward: bool = True + tie_word_embeddings_support: TieSupport = TieSupport.BOTH + _tied_weights_keys = {"lm_head.weight": "model.language_model.embed_tokens.weight"} + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" @@ -831,6 +843,7 @@ def __init__( num_nextn_predict_layers: int | None = None, **kwargs: Any, ) -> None: + reject_unsupported_tie_word_embeddings(type(self), config) del kwargs super().__init__(config) self.backend = _qwen3_5_backend(backend) @@ -859,6 +872,10 @@ def __init__( # final hidden states and ``lm_head`` agree. if self.lm_head is not None and self.lm_head.weight.dtype != dtype: self.lm_head = self.lm_head.to(dtype) + # HF post_init tied lm_head to the pre-swap embedding; the language_model + # swap above orphaned that alias, so re-tie to the active embedding when the + # config requests it (no-op otherwise). + self.tie_weights() self.mtp_config = build_mtp_config_from_hf( text_config, loss_scaling_factor=mtp_loss_scaling_factor, @@ -872,6 +889,11 @@ def __init__( if self.backend.enable_hf_state_dict_adapter: self.state_dict_adapter = Qwen3_5DenseStateDictAdapter(route_linear_attn_fp32_params=True) + def tie_weights(self, *_args: object, **_kwargs: object) -> None: + """Tie ``lm_head`` to the active VLM text embedding when requested.""" + if getattr(self.config, "tie_word_embeddings", False): + self.lm_head.weight = self.model.language_model.embed_tokens.weight + def _pop_staged_vlm_media( self, input_ids: torch.Tensor | None, diff --git a/nemo_automodel/components/models/qwen3_5_moe/model.py b/nemo_automodel/components/models/qwen3_5_moe/model.py index 0d588be584..c74866debc 100644 --- a/nemo_automodel/components/models/qwen3_5_moe/model.py +++ b/nemo_automodel/components/models/qwen3_5_moe/model.py @@ -60,10 +60,13 @@ def _make_missing(name: str): Qwen3_5MoeVisionRotaryEmbedding = _make_missing("Qwen3_5MoeVisionRotaryEmbedding") HFQwen3_5MoeModel = _make_missing("Qwen3_5MoeModel") -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import BackendConfig, initialize_linear_module from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin from nemo_automodel.components.models.common.mtp import MTPConfig, MTPModule, roll_tensor +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.qwen3_next.layers import Qwen3NextRMSNorm from nemo_automodel.components.models.qwen3_next.model import Block @@ -693,6 +696,8 @@ class Qwen3_5MoeForConditionalGeneration(HFCheckpointingMixin, HFQwen3_5MoeForCo * ``lm_head`` with NeMo backend linear """ + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + # forward() pulls per-microbatch pixel_values from _vlm_pixel_values_chunks; # patch_hf_model_for_pp must not replace it under PP. _pp_keep_self_forward: bool = True @@ -754,7 +759,7 @@ def __init__( if sub_cfg is not config and hasattr(sub_cfg, "torch_dtype"): sub_cfg.torch_dtype = top_dtype - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) # Initialize HF parent (creates self.model, self.lm_head, vision encoder, etc.) super().__init__(config) diff --git a/nemo_automodel/components/models/qwen3_moe/model.py b/nemo_automodel/components/models/qwen3_moe/model.py index 18f36988b6..5074305d94 100644 --- a/nemo_automodel/components/models/qwen3_moe/model.py +++ b/nemo_automodel/components/models/qwen3_moe/model.py @@ -20,7 +20,6 @@ from transformers.modeling_outputs import CausalLMOutputWithPast from transformers.models.qwen3_moe.configuration_qwen3_moe import Qwen3MoeConfig -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.distributed.activation_checkpointing import unwrap_checkpoint_wrapper from nemo_automodel.components.models.common import ( BackendConfig, @@ -29,6 +28,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.gpt_oss.rope_utils import RotaryEmbedding, position_ids_to_freqs_cis from nemo_automodel.components.models.qwen3_moe.layers import Qwen3MoeAttention @@ -244,6 +247,7 @@ class Qwen3MoeForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): # The generic HF pipeline forward assumes the HF rotary API # (rotary_emb(x, position_ids) -> cos/sin) and crashes on the freqs_cis / # THD / CP path, so it must not clobber our forward. + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY _pp_keep_self_forward: bool = True @dataclass(frozen=True) @@ -284,7 +288,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) self.model = Qwen3MoeModel(config, backend=self.backend, moe_config=moe_config, moe_overrides=moe_overrides) diff --git a/nemo_automodel/components/models/qwen3_next/model.py b/nemo_automodel/components/models/qwen3_next/model.py index 9978ce888a..56582d7d0f 100644 --- a/nemo_automodel/components/models/qwen3_next/model.py +++ b/nemo_automodel/components/models/qwen3_next/model.py @@ -20,7 +20,6 @@ from transformers.modeling_outputs import CausalLMOutputWithPast from transformers.models.qwen3_next.configuration_qwen3_next import Qwen3NextConfig -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.distributed.activation_checkpointing import unwrap_checkpoint_wrapper from nemo_automodel.components.models.common import ( BackendConfig, @@ -29,6 +28,10 @@ initialize_rms_norm_module, ) from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.gpt_oss.rope_utils import RotaryEmbedding, position_ids_to_freqs_cis from nemo_automodel.components.models.qwen3_next.layers import ( @@ -263,6 +266,8 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class Qwen3NextForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" @@ -301,7 +306,7 @@ def __init__( ): super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) self.model = Qwen3NextModel(config, backend=self.backend, moe_config=moe_config, moe_overrides=moe_overrides) diff --git a/nemo_automodel/components/models/qwen3_omni_moe/model.py b/nemo_automodel/components/models/qwen3_omni_moe/model.py index dc65fc39f9..d2f89a8bd3 100644 --- a/nemo_automodel/components/models/qwen3_omni_moe/model.py +++ b/nemo_automodel/components/models/qwen3_omni_moe/model.py @@ -29,9 +29,12 @@ Qwen3OmniMoeThinkerTextRotaryEmbedding as HFQwen3OmniMoeThinkerTextRotaryEmbedding, ) -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import BackendConfig, initialize_linear_module, initialize_rms_norm_module from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.qwen3_moe.model import Block from nemo_automodel.components.models.qwen3_omni_moe.state_dict_adapter import Qwen3OmniMoeStateDictAdapter @@ -220,6 +223,8 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( ): """Qwen3OmniMoe Thinker for Conditional Generation with multimodal support.""" + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + @dataclass(frozen=True) class ModelCapabilities: """Declared parallelism capabilities for this model class.""" @@ -258,7 +263,7 @@ def __init__( ): base_config = config.thinker_config if hasattr(config, "thinker_config") else config backend = backend or BackendConfig() - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) # _init_model() only overrides the top-level hf_config.torch_dtype; for # Omni configs the real params live under thinker_config.text_config / diff --git a/nemo_automodel/components/models/qwen3_vl_moe/model.py b/nemo_automodel/components/models/qwen3_vl_moe/model.py index 514b82716c..9058fcacae 100644 --- a/nemo_automodel/components/models/qwen3_vl_moe/model.py +++ b/nemo_automodel/components/models/qwen3_vl_moe/model.py @@ -30,9 +30,12 @@ Qwen3VLMoeVisionRotaryEmbedding, ) -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import BackendConfig, initialize_linear_module, initialize_rms_norm_module from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.qwen3_moe.model import Block from nemo_automodel.components.moe.config import MoEConfig @@ -445,6 +448,8 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class Qwen3VLMoeForConditionalGeneration(HFCheckpointingMixin, HFQwen3VLMoeForConditionalGeneration, MoEFSDPSyncMixin): """Qwen3-VL conditional generation model using the Qwen3-MoE backend components.""" + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + # forward() pulls per-microbatch pixel_values from _vlm_pixel_values_chunks; # patch_hf_model_for_pp must not replace it under PP. _pp_keep_self_forward: bool = True @@ -500,7 +505,7 @@ def __init__( if sub_cfg is not config and hasattr(sub_cfg, "torch_dtype"): sub_cfg.torch_dtype = top_dtype - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) super().__init__(config) self.backend = backend diff --git a/nemo_automodel/components/models/step3p5/model.py b/nemo_automodel/components/models/step3p5/model.py index c5b9146890..3aeba8cda5 100644 --- a/nemo_automodel/components/models/step3p5/model.py +++ b/nemo_automodel/components/models/step3p5/model.py @@ -19,9 +19,12 @@ import torch.nn as nn from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import BackendConfig, get_rope_config, initialize_linear_module from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.gpt_oss.rope_utils import RotaryEmbedding, position_ids_to_freqs_cis from nemo_automodel.components.models.step3p5.layers import ( @@ -402,6 +405,8 @@ def init_weights(self, buffer_device: torch.device | None = None) -> None: class Step3p5ForCausalLM(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): """Step3p5 model for causal language modeling.""" + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + _keep_in_fp32_modules = ["rotary_emb"] @dataclass(frozen=True) @@ -451,7 +456,7 @@ def __init__( ) -> None: super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) self.model = Step3p5Model(config, backend=self.backend, moe_config=moe_config, moe_overrides=moe_overrides) diff --git a/nemo_automodel/components/models/step3p7/model.py b/nemo_automodel/components/models/step3p7/model.py index fc6a37e3d1..1ec18a605b 100644 --- a/nemo_automodel/components/models/step3p7/model.py +++ b/nemo_automodel/components/models/step3p7/model.py @@ -23,10 +23,13 @@ import torch.nn as nn from transformers.modeling_outputs import CausalLMOutputWithPast -from nemo_automodel.components.checkpoint.utils import reject_unsupported_tied_word_embeddings from nemo_automodel.components.models.common import BackendConfig, initialize_linear_module from nemo_automodel.components.models.common.hf_checkpointing_mixin import HFCheckpointingMixin from nemo_automodel.components.models.common.mtp import roll_tensor +from nemo_automodel.components.models.common.tie_word_embeddings import ( + TieSupport, + reject_unsupported_tie_word_embeddings, +) from nemo_automodel.components.models.common.utils import cast_model_to_dtype, compute_lm_head_logits from nemo_automodel.components.models.gpt_oss.rope_utils import position_ids_to_freqs_cis from nemo_automodel.components.models.step3p5.model import Step3p5Model @@ -313,6 +316,8 @@ def forward( class Step3p7ForConditionalGeneration(HFCheckpointingMixin, nn.Module, MoEFSDPSyncMixin): """Native Step3.7 VLM implementation for MedPix fine-tuning with EP and PP.""" + tie_word_embeddings_support: TieSupport = TieSupport.UNTIED_ONLY + _keep_in_fp32_modules = ["rotary_emb"] _pp_keep_self_forward: bool = True @@ -356,7 +361,7 @@ def __init__( ) -> None: super().__init__() self.config = config - reject_unsupported_tied_word_embeddings(config, type(self).__name__) + reject_unsupported_tie_word_embeddings(type(self), config) self.backend = backend or BackendConfig() moe_overrides = kwargs.pop("moe_overrides", None) mtp_loss_scaling_factor = kwargs.pop("mtp_loss_scaling_factor", 0.1) diff --git a/skills/nemo-automodel-model-onboarding/SKILL.md b/skills/nemo-automodel-model-onboarding/SKILL.md index fab9152941..412875d6ed 100644 --- a/skills/nemo-automodel-model-onboarding/SKILL.md +++ b/skills/nemo-automodel-model-onboarding/SKILL.md @@ -104,7 +104,7 @@ Download the model's `config.json` from the HuggingFace Hub (or use `AutoConfig. - `model_type` -- used for custom config registration in `_CUSTOM_CONFIG_REGISTRATIONS` if HF does not have a built-in config class - `hidden_size`, `intermediate_size`, `num_hidden_layers`, `num_attention_heads`, `num_key_value_heads` -- sizing - `vocab_size` -- needed for tiny test configs -- `tie_word_embeddings` -- whether lm_head shares weights with embed_tokens +- `tie_word_embeddings` -- the saved setting in each supported checkpoint; do not infer it from a bare config constructor - `hidden_act` -- activation function (e.g., `"silu"` for SwiGLU) ### 1.2 Determine model type @@ -195,11 +195,47 @@ See the pattern files for detailed implementation guidance: ### 2.3 Causal LM weight tying -For any CausalLM-style class whose config can enable `tie_word_embeddings`, -make tying explicit: declare `_tied_weights_keys`, implement `tie_weights()` -with the actual `lm_head` and input-embedding FQNs, and add tiny tests for -tied and untied configs. Do not tie architectures with intentionally separate -heads, asymmetric vocab sizes, or stages that do not own both tensors. +Every registered model class with a causal `lm_head` must: + +- Declare `tie_word_embeddings_support: TieSupport` as `BOTH`, `TIED_ONLY`, or + `UNTIED_ONLY`. +- Call `reject_unsupported_tie_word_embeddings(type(self), config)` at the top + of `__init__`, using the original config before unwrapping `text_config` or + `thinker_config`. + +Only classes with no causal LM head may be explicitly exempted from the registry +test. + +Choose the policy from the implementation and the actual supported checkpoint +configs, not from a bare config constructor: + +- `BOTH`: tied and untied configurations are both supported. +- `TIED_ONLY`: only a tied configuration is supported. +- `UNTIED_ONLY`: only an untied configuration is supported. + +Runtime helpers must treat `TIED_ONLY` and `UNTIED_ONLY` as authoritative and +only resolve a per-checkpoint config flag for `BOTH`. All current `BOTH` VLMs +honor the outer `tie_word_embeddings` flag, so do not add a model-specific +resolver until a supported `BOTH` model actually requires another config path. + +For `BOTH` and `TIED_ONLY`, always declare `_tied_weights_keys` and implement +`tie_weights()` with the actual `lm_head` and input-embedding FQNs. Do not rely +on inherited Hugging Face tying, and re-tie after any language-model swap. + +Add policy-specific tests: + +- `BOTH`: tied aliases; untied does not alias. +- `TIED_ONLY`: tied aliases; untied is rejected. +- `UNTIED_ONLY`: weights stay separate; tied is rejected. + +Do not tie architectures with intentionally separate heads, asymmetric vocab +sizes, or stages that do not own both tensors. + +For `from_pretrained`, the checkpoint's saved `tie_word_embeddings` value is +authoritative, even for `BOTH`. The `NeMoAuto*` bridge rejects flips in either +direction. A model-owned `from_pretrained` that bypasses that bridge must call +`reject_tie_word_embeddings_flip(checkpoint_config, requested_config, +model_class_name)`. ### 2.4 MoE state-dict adapter checklist @@ -395,6 +431,9 @@ that only surface in a full parity comparison. - [ ] Registered in `MODEL_ARCH_MAPPING` in `_transformers/registry.py` - [ ] Registered custom config in `_CUSTOM_CONFIG_REGISTRATIONS` (if applicable) - [ ] Declared `ModelCapabilities` nested dataclass (static) OR `get_capabilities(cls, config)` classmethod (variant dispatch, e.g. ERNIE-4.5 MoE vs dense) — never both, never neither +- [ ] Declared `TieSupport` and called the constructor guard for every class with a causal `lm_head` (or added an explicit no-head exemption) -- see §2.3 +- [ ] Added explicit `_tied_weights_keys` and `tie_weights()` for `BOTH` / `TIED_ONLY`, plus policy-specific alias and rejection tests -- see §2.3 +- [ ] Guarded any model-owned `from_pretrained` that bypasses the `NeMoAuto*` bridge against checkpoint flips -- see §2.3 - [ ] Created example YAML config - [ ] Verified model loads via `NeMoAutoModelForCausalLM.from_pretrained()` - [ ] Created unit tests (forward shape, state_dict round-trip) diff --git a/tests/functional_tests/llm_pretrain_and_kd/run_tp_output_parity_minified.py b/tests/functional_tests/llm_pretrain_and_kd/run_tp_output_parity_minified.py index 5676d5ea2f..e150d1ed14 100644 --- a/tests/functional_tests/llm_pretrain_and_kd/run_tp_output_parity_minified.py +++ b/tests/functional_tests/llm_pretrain_and_kd/run_tp_output_parity_minified.py @@ -254,7 +254,7 @@ def _build_minified_model(kind: ModelKind): head_dim=16, max_position_embeddings=128, use_cache=False, - tie_word_embeddings=True, + tie_word_embeddings=False, rope_parameters={ "type": "yarn", "rope_theta": 1000000.0, diff --git a/tests/unit_tests/_transformers/test_auto_model.py b/tests/unit_tests/_transformers/test_auto_model.py index 89f9b84e26..bb3869dbfc 100644 --- a/tests/unit_tests/_transformers/test_auto_model.py +++ b/tests/unit_tests/_transformers/test_auto_model.py @@ -28,6 +28,7 @@ _consume_config_overrides, _get_next_fallback_attn, _init_model, + _maybe_reject_tie_word_embeddings_flip, _patch_attention, _patch_remote_code_compat, _resolve_distributed_setup, @@ -375,10 +376,13 @@ def fake_hook(model, mesh): fake_module = types.SimpleNamespace(apply_model_runtime_patches=fake_hook) test_registry = {"FakeArchForCausalLM": ("fake.module.path", "apply_model_runtime_patches")} - with patch.object(kp, "_MODEL_RUNTIME_PATCHES", test_registry), patch( - "nemo_automodel._transformers.kernel_patches.importlib.import_module", - return_value=fake_module, - ) as mock_import: + with ( + patch.object(kp, "_MODEL_RUNTIME_PATCHES", test_registry), + patch( + "nemo_automodel._transformers.kernel_patches.importlib.import_module", + return_value=fake_module, + ) as mock_import, + ): assert apply_model_runtime_patches(model, mesh) is model mock_import.assert_called_once_with("fake.module.path") @@ -401,9 +405,12 @@ def fake_hook(model, mesh): shared_spec = ("fake.module.path", "apply_model_runtime_patches") test_registry = {"FakeArchA": shared_spec, "FakeArchB": shared_spec} - with patch.object(kp, "_MODEL_RUNTIME_PATCHES", test_registry), patch( - "nemo_automodel._transformers.kernel_patches.importlib.import_module", - return_value=fake_module, + with ( + patch.object(kp, "_MODEL_RUNTIME_PATCHES", test_registry), + patch( + "nemo_automodel._transformers.kernel_patches.importlib.import_module", + return_value=fake_module, + ), ): assert apply_model_runtime_patches(model, mesh) is model @@ -1975,3 +1982,67 @@ def test_compatible_model_unaffected(self): # Existing values should be preserved assert model.all_tied_weights_keys == ["existing.key"] assert model.config.use_cache is True + + +class TestMaybeRejectTieWordEmbeddingsFlip: + """Layer 2 from_pretrained flip guard (_maybe_reject_tie_word_embeddings_flip). + + from_pretrained accepts str | os.PathLike sources, so the guard must normalize + path-like inputs with os.fspath() and enforce the checkpoint's raw + tie_word_embeddings for both — a pathlib.Path local checkpoint must not bypass + the check. Non-path sources are skipped and a failed raw-config re-read is + conservative (never blocks the load). + """ + + @staticmethod + def _requested_config(tied): + return types.SimpleNamespace(tie_word_embeddings=tied, architectures=["DummyForCausalLM"]) + + @staticmethod + def _patch_raw_config(**kwargs): + return patch( + "nemo_automodel._transformers.auto_model.AutoConfig.from_pretrained", + **kwargs, + ) + + def test_str_source_flip_rejected(self): + raw = types.SimpleNamespace(tie_word_embeddings=True) + with self._patch_raw_config(return_value=raw): + with pytest.raises(NotImplementedError, match="flipping the flag is not supported"): + _maybe_reject_tie_word_embeddings_flip("org/tied-model", self._requested_config(tied=False), {}) + + def test_pathlib_path_source_flip_rejected(self): + from pathlib import Path + + raw = types.SimpleNamespace(tie_word_embeddings=True) + with self._patch_raw_config(return_value=raw): + with pytest.raises(NotImplementedError, match="flipping the flag is not supported"): + _maybe_reject_tie_word_embeddings_flip( + Path("/ckpts/tied-model"), self._requested_config(tied=False), {} + ) + + def test_pathlib_path_source_matching_value_passes(self): + from pathlib import Path + + raw = types.SimpleNamespace(tie_word_embeddings=True) + with self._patch_raw_config(return_value=raw): + _maybe_reject_tie_word_embeddings_flip(Path("/ckpts/tied-model"), self._requested_config(tied=True), {}) + + def test_pathlib_path_normalized_to_str_for_raw_read(self): + from pathlib import Path + + raw = types.SimpleNamespace(tie_word_embeddings=True) + with self._patch_raw_config(return_value=raw) as mock_from_pretrained: + _maybe_reject_tie_word_embeddings_flip(Path("/ckpts/tied-model"), self._requested_config(tied=True), {}) + (source,) = mock_from_pretrained.call_args.args + assert isinstance(source, str) + assert source == str(Path("/ckpts/tied-model")) + + def test_non_path_source_skipped(self): + with self._patch_raw_config() as mock_from_pretrained: + _maybe_reject_tie_word_embeddings_flip(None, self._requested_config(tied=False), {}) + mock_from_pretrained.assert_not_called() + + def test_raw_config_read_failure_does_not_block(self): + with self._patch_raw_config(side_effect=OSError("offline")): + _maybe_reject_tie_word_embeddings_flip("org/unreachable", self._requested_config(tied=False), {}) diff --git a/tests/unit_tests/_transformers/test_model_init.py b/tests/unit_tests/_transformers/test_model_init.py index 553d6585d9..3dd4827988 100644 --- a/tests/unit_tests/_transformers/test_model_init.py +++ b/tests/unit_tests/_transformers/test_model_init.py @@ -745,6 +745,18 @@ def test_untied_config_keeps_separate_lm_head(self): assert model.lm_head.weight.data_ptr() != model.model.embed_tokens.weight.data_ptr() torch.testing.assert_close(model.lm_head.weight, lm_head_before) + def test_untied_only_policy_overrides_misleading_tied_config(self): + """A fixed untied policy prevents re-tying despite an outer True flag.""" + from nemo_automodel._transformers.model_init import _tie_weights_nemo + from nemo_automodel.components.models.common.tie_word_embeddings import TieSupport + + model = self._make_model(tie=True) + model.tie_word_embeddings_support = TieSupport.UNTIED_ONLY + + _tie_weights_nemo(model) + + assert model.lm_head.weight is not model.model.embed_tokens.weight + def test_tied_config_reties(self): """tie_word_embeddings=True: keep the #1817 re-tie behavior.""" from nemo_automodel._transformers.model_init import _tie_weights_nemo diff --git a/tests/unit_tests/_transformers/test_tie_support_registry.py b/tests/unit_tests/_transformers/test_tie_support_registry.py new file mode 100644 index 0000000000..6f673ca371 --- /dev/null +++ b/tests/unit_tests/_transformers/test_tie_support_registry.py @@ -0,0 +1,82 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Registry guardrail: every head-owning registered model declares a TieSupport policy. + +``reject_unsupported_tie_word_embeddings`` defaults *undeclared* classes to +``TieSupport.BOTH``, so this test is the only enforcement that a newly onboarded +causal-LM / conditional-generation model cannot silently skip declaring +``tie_word_embeddings_support``. Every ``MODEL_ARCH_MAPPING`` class must either +declare a policy or be listed in the explicit, commented exemption below. +""" + +import importlib + +from nemo_automodel._transformers.registry import MODEL_ARCH_MAPPING +from nemo_automodel.components.models.common.tie_word_embeddings import TieSupport + +# Registered classes that do NOT own a causal LM head and are therefore exempt from +# declaring ``tie_word_embeddings_support``. Keep this explicit and commented so a +# head-owning model cannot be onboarded without either a declaration or a deliberate +# exemption reviewed here. +_TIE_SUPPORT_EXEMPT: dict[str, str] = { + "LlamaBidirectionalModel": "retrieval bidirectional encoder; no lm_head", + "LlamaBidirectionalForSequenceClassification": "retrieval; sequence-classification head, not an lm_head", + "Ministral3BidirectionalModel": "retrieval bidirectional encoder; no lm_head", + "LlamaNemotronVLModel": "retrieval VL encoder; no causal lm_head", +} + + +def _registered_classes(): + """Yield ``(arch, module_path, class_name)`` for each distinct registered class.""" + seen: set[str] = set() + for arch, spec in MODEL_ARCH_MAPPING.items(): + module_path, class_name = spec[0], spec[1] + if class_name in seen: + continue + seen.add(class_name) + yield arch, module_path, class_name + + +def test_every_registered_lm_head_class_declares_tie_support(): + checked = 0 + for arch, module_path, class_name in _registered_classes(): + if class_name in _TIE_SUPPORT_EXEMPT: + continue + try: + cls = getattr(importlib.import_module(module_path), class_name) + except Exception: + # Optional-dependency-gated arch not importable in this environment; it is + # exercised in the fuller CI image. Skip rather than raise a false failure. + continue + support = getattr(cls, "tie_word_embeddings_support", None) + assert isinstance(support, TieSupport), ( + f"{class_name} (registered as {arch!r}) does not declare a TieSupport policy. " + f"Add `tie_word_embeddings_support: TieSupport = TieSupport.` " + f"to the class, or add it to _TIE_SUPPORT_EXEMPT with a reason if it owns no causal lm_head." + ) + if support in (TieSupport.BOTH, TieSupport.TIED_ONLY): + assert callable(cls.__dict__.get("tie_weights")), ( + f"{class_name} declares {support.name} but does not define a model-local tie_weights(). " + "Implement the exact lm_head/input-embedding alias instead of relying on an inherited HF method." + ) + checked += 1 + assert checked > 0, "no registered classes were checked — MODEL_ARCH_MAPPING import likely broke" + + +def test_tie_support_exempt_list_has_no_stale_entries(): + """Every exempt name must still be a registered class (catch renames/removals).""" + registered = {class_name for _, _, class_name in _registered_classes()} + stale = set(_TIE_SUPPORT_EXEMPT) - registered + assert not stale, f"stale exempt entries no longer in MODEL_ARCH_MAPPING: {sorted(stale)}" diff --git a/tests/unit_tests/distributed/pipelining/test_hf_utils.py b/tests/unit_tests/distributed/pipelining/test_hf_utils.py index 23a25446ec..20325204c8 100644 --- a/tests/unit_tests/distributed/pipelining/test_hf_utils.py +++ b/tests/unit_tests/distributed/pipelining/test_hf_utils.py @@ -554,7 +554,7 @@ def __init__(self): model = MockModel() - with pytest.raises(ValueError, match="tie_word_embeddings=True is not supported"): + with pytest.raises(ValueError, match="Pipeline parallelism does not support tie_word_embeddings=True"): validate_hf_model_for_pipeline_support(model) def test_validate_encoder_decoder_model(self): diff --git a/tests/unit_tests/models/bagel/test_bagel_understanding.py b/tests/unit_tests/models/bagel/test_bagel_understanding.py index 8642879a58..3d945cddd5 100644 --- a/tests/unit_tests/models/bagel/test_bagel_understanding.py +++ b/tests/unit_tests/models/bagel/test_bagel_understanding.py @@ -23,6 +23,8 @@ import inspect +import pytest + def test_bagel_imports() -> None: from nemo_automodel.components.models.bagel import ( @@ -59,3 +61,23 @@ def test_bagel_stage1_config_drops_generation_path() -> None: assert cfg.visual_gen is False assert cfg.text_config.layer_module == "Qwen2DecoderLayer" + + +def test_bagel_rejects_tied_word_embeddings() -> None: + from nemo_automodel.components.models.bagel.configuration import BagelConfig + from nemo_automodel.components.models.bagel.model import BagelForUnifiedMultimodal + + # UNTIED_ONLY: the guard reads the nested text_config tie flag and raises at the + # top of __init__, before the (checkpoint-sized) model is constructed. + cfg = BagelConfig( + text_config=dict( + tie_word_embeddings=True, + hidden_size=16, + num_attention_heads=2, + num_hidden_layers=1, + vocab_size=32, + intermediate_size=32, + ) + ) + with pytest.raises(NotImplementedError, match="does not support tie_word_embeddings=True"): + BagelForUnifiedMultimodal(cfg) diff --git a/tests/unit_tests/models/baichuan/test_baichuan_model.py b/tests/unit_tests/models/baichuan/test_baichuan_model.py index cb3bf5a7fa..f34cf98533 100644 --- a/tests/unit_tests/models/baichuan/test_baichuan_model.py +++ b/tests/unit_tests/models/baichuan/test_baichuan_model.py @@ -520,3 +520,12 @@ def test_reorders_correctly(self): for layer_past in reordered: assert layer_past[0].shape == (3, 2, 4, 8) assert torch.allclose(reordered[0][0][0], past[0][0][2]) + + +class TestBaichuanTieWordEmbeddings: + """Baichuan is UNTIED_ONLY: its NormHead lm_head cannot be meaningfully tied.""" + + def test_rejects_tied_word_embeddings(self): + # Guard raises at the top of __init__ (before NormHead construction). + with pytest.raises(NotImplementedError, match="does not support tie_word_embeddings=True"): + BaichuanForCausalLM(_tiny_config(tie_word_embeddings=True)) diff --git a/tests/unit_tests/models/diffusion_gemma/test_diffusion_gemma_model.py b/tests/unit_tests/models/diffusion_gemma/test_diffusion_gemma_model.py index 999997d380..d2fac30e6d 100644 --- a/tests/unit_tests/models/diffusion_gemma/test_diffusion_gemma_model.py +++ b/tests/unit_tests/models/diffusion_gemma/test_diffusion_gemma_model.py @@ -100,6 +100,16 @@ def test_construct_and_forward_shape(): assert torch.isfinite(out.logits).all() +def test_tie_weights_restores_lm_head_alias(): + model, _ = _tiny_model() + model.lm_head.weight = torch.nn.Parameter(model.lm_head.weight.detach().clone()) + assert model.lm_head.weight is not model.model.embed_tokens.weight + + model.tie_weights() + + assert model.lm_head.weight is model.model.embed_tokens.weight + + def test_top_forward_enters_backbone_call_for_fsdp_hooks(): torch.manual_seed(0) model, _ = _tiny_model() diff --git a/tests/unit_tests/models/ernie4_5/test_ernie4_5_model.py b/tests/unit_tests/models/ernie4_5/test_ernie4_5_model.py index 99b148f513..3719846b5a 100644 --- a/tests/unit_tests/models/ernie4_5/test_ernie4_5_model.py +++ b/tests/unit_tests/models/ernie4_5/test_ernie4_5_model.py @@ -327,10 +327,11 @@ def test_lm_head_tied(self, dense_config, backend_config): model = Ernie4_5ForCausalLM(dense_config, backend=backend_config) assert model.lm_head.weight is model.model.embed_tokens.weight - def test_lm_head_untied(self, dense_config, backend_config): + def test_lm_head_untied_is_rejected(self, dense_config, backend_config): + # ERNIE-4.5 is TIED_ONLY: both shipped checkpoints are tied, so untying is rejected. dense_config.tie_word_embeddings = False - model = Ernie4_5ForCausalLM(dense_config, backend=backend_config) - assert model.lm_head.weight is not model.model.embed_tokens.weight + with pytest.raises(NotImplementedError, match="does not support tie_word_embeddings=False"): + Ernie4_5ForCausalLM(dense_config, backend=backend_config) def test_tie_weights_rebinds(self, dense_config, backend_config): dense_config.tie_word_embeddings = True @@ -388,10 +389,11 @@ def test_lm_head_tied(self, moe_hf_config, backend_config): model = Ernie4_5_MoeForCausalLM(moe_hf_config, backend=backend_config) assert model.lm_head.weight is model.model.embed_tokens.weight - def test_lm_head_untied(self, moe_hf_config, backend_config): + def test_lm_head_untied_is_rejected(self, moe_hf_config, backend_config): + # ERNIE-4.5 is TIED_ONLY: both shipped checkpoints are tied, so untying is rejected. moe_hf_config.tie_word_embeddings = False - model = Ernie4_5_MoeForCausalLM(moe_hf_config, backend=backend_config) - assert model.lm_head.weight is not model.model.embed_tokens.weight + with pytest.raises(NotImplementedError, match="does not support tie_word_embeddings=False"): + Ernie4_5_MoeForCausalLM(moe_hf_config, backend=backend_config) def test_state_dict_adapter_off_by_default(self, moe_hf_config, backend_config): model = Ernie4_5_MoeForCausalLM(moe_hf_config, backend=backend_config) diff --git a/tests/unit_tests/models/gemma4_drafter/test_drafter_wrapper.py b/tests/unit_tests/models/gemma4_drafter/test_drafter_wrapper.py index 472cd6b8b6..7c14f72da5 100644 --- a/tests/unit_tests/models/gemma4_drafter/test_drafter_wrapper.py +++ b/tests/unit_tests/models/gemma4_drafter/test_drafter_wrapper.py @@ -147,6 +147,25 @@ def test_lm_head_tied_to_embed_tokens(self): assert cfg.tie_word_embeddings is True assert model.lm_head.weight.data_ptr() == model.model.embed_tokens.weight.data_ptr() + def test_tie_weights_restores_lm_head_alias(self): + from transformers.models.gemma4_assistant.configuration_gemma4_assistant import ( + Gemma4AssistantConfig, + ) + + from nemo_automodel.components.models.gemma4_drafter.model import ( + Gemma4DrafterForCausalLM, + ) + + text_cfg = _make_tiny_drafter_text_config() + cfg = Gemma4AssistantConfig(text_config=text_cfg, backbone_hidden_size=text_cfg.hidden_size) + model = Gemma4DrafterForCausalLM(cfg) + model.lm_head.weight = torch.nn.Parameter(model.lm_head.weight.detach().clone()) + assert model.lm_head.weight is not model.model.embed_tokens.weight + + model.tie_weights() + + assert model.lm_head.weight is model.model.embed_tokens.weight + def test_use_ordered_embeddings_creates_masked_embedder(self): from transformers.models.gemma4_assistant.configuration_gemma4_assistant import ( Gemma4AssistantConfig, diff --git a/tests/unit_tests/models/mistral3/test_mistral3_model.py b/tests/unit_tests/models/mistral3/test_mistral3_model.py index bf6df2b6c7..6ff8f8e2bc 100644 --- a/tests/unit_tests/models/mistral3/test_mistral3_model.py +++ b/tests/unit_tests/models/mistral3/test_mistral3_model.py @@ -14,6 +14,7 @@ from unittest.mock import patch +import pytest import torch from transformers import AutoConfig, AutoModel from transformers.modeling_outputs import BaseModelOutputWithPast @@ -26,7 +27,7 @@ ) -def tiny_config() -> Ministral3Config: +def tiny_config(**overrides) -> Ministral3Config: cfg = Ministral3Config( vocab_size=32, hidden_size=16, @@ -37,6 +38,7 @@ def tiny_config() -> Ministral3Config: head_dim=8, max_position_embeddings=64, attention_dropout=0.0, + **overrides, ) # Ensure eager attention path in tests to avoid optional backends. cfg._attn_implementation = "eager" @@ -86,6 +88,11 @@ def test_forward_runs_layers_and_returns_last_hidden_state(self): class TestMinistral3ForCausalLM: + def test_rejects_tied_word_embeddings(self): + # UNTIED_ONLY: the guard raises at the top of __init__ before construction. + with pytest.raises(NotImplementedError, match="does not support tie_word_embeddings=True"): + Ministral3ForCausalLM(tiny_config(tie_word_embeddings=True)) + def test_forward_emits_logits(self): cfg = tiny_config() model = Ministral3ForCausalLM(cfg) diff --git a/tests/unit_tests/models/mistral3_vlm/test_model.py b/tests/unit_tests/models/mistral3_vlm/test_model.py index 7f71b05a31..be1326300e 100644 --- a/tests/unit_tests/models/mistral3_vlm/test_model.py +++ b/tests/unit_tests/models/mistral3_vlm/test_model.py @@ -29,7 +29,9 @@ import pytest import torch import torch.nn as nn +from transformers import Mistral3Config +from nemo_automodel.components.models.common.tie_word_embeddings import TieSupport, reject_tie_word_embeddings_flip from nemo_automodel.components.models.mistral3_vlm.model import ( Mistral3FP8VLMForConditionalGeneration, _rotary_reinit_self_hook, @@ -249,3 +251,78 @@ def __init__(self, config, device=None): # Should not raise. _rotary_reinit_self_hook(rot, args=(), kwargs={}) assert rot._mistral3_fp8_rotary_reinit_done is True + + +# --------------------------------------------------------------------------- # +# tie_word_embeddings (BOTH: serves tied Ministral-3 + untied Mistral-Medium) # +# --------------------------------------------------------------------------- # +def _tiny_vlm_config(tie_word_embeddings: bool) -> Mistral3Config: + """Tiny Mistral3 VLM config (2 text + 2 vision layers) for CPU construction.""" + return Mistral3Config( + text_config=dict( + model_type="mistral", + hidden_size=64, + intermediate_size=128, + num_hidden_layers=2, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=16, + vocab_size=256, + max_position_embeddings=128, + tie_word_embeddings=tie_word_embeddings, + ), + vision_config=dict( + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=4, + image_size=32, + patch_size=16, + head_dim=8, + ), + tie_word_embeddings=tie_word_embeddings, + image_token_index=1, + ) + + +class TestTieWordEmbeddings: + """One class loads both tied (Ministral-3, lm_head not serialized) and untied + (Mistral-Medium-3.5-128B, Devstral-24B) checkpoints, so it declares BOTH and + construction must honor the config flag either way — no construction-time + rejection. Per-checkpoint enforcement (rejecting a flag flipped away from the + checkpoint's value) is the from_pretrained flip guard's job, covered by + test_from_pretrained_flip_guard_rejects_tie_mismatch below. + """ + + def test_declares_both(self): + assert Mistral3FP8VLMForConditionalGeneration.tie_word_embeddings_support is TieSupport.BOTH + + def test_tied_config_shares_lm_head_storage(self): + model = Mistral3FP8VLMForConditionalGeneration(_tiny_vlm_config(tie_word_embeddings=True)) + assert model.get_input_embeddings().weight is model.get_output_embeddings().weight + + def test_tie_weights_restores_tied_alias(self): + model = Mistral3FP8VLMForConditionalGeneration(_tiny_vlm_config(tie_word_embeddings=True)) + model.lm_head.weight = nn.Parameter(model.lm_head.weight.detach().clone()) + assert model.get_input_embeddings().weight is not model.get_output_embeddings().weight + + model.tie_weights() + + assert model.get_input_embeddings().weight is model.get_output_embeddings().weight + + def test_untied_config_has_separate_lm_head(self): + model = Mistral3FP8VLMForConditionalGeneration(_tiny_vlm_config(tie_word_embeddings=False)) + assert model.get_input_embeddings().weight is not model.get_output_embeddings().weight + + def test_from_pretrained_flip_guard_rejects_tie_mismatch(self): + # Layer 2: as a BOTH class it relies on the from_pretrained flip guard to enforce + # each checkpoint's own tie value. The resolver reads the top-level flag for this + # class, so a top-level mismatch must be rejected in either direction. + cls_name = "Mistral3FP8VLMForConditionalGeneration" + untied = SimpleNamespace(tie_word_embeddings=False) + tied = SimpleNamespace(tie_word_embeddings=True) + with pytest.raises(NotImplementedError, match="flipping the flag is not supported"): + reject_tie_word_embeddings_flip(untied, tied, cls_name) + with pytest.raises(NotImplementedError, match="flipping the flag is not supported"): + reject_tie_word_embeddings_flip(tied, untied, cls_name) + reject_tie_word_embeddings_flip(untied, untied, cls_name) # matching value -> no raise diff --git a/tests/unit_tests/models/mistral4/test_mistral4_model.py b/tests/unit_tests/models/mistral4/test_mistral4_model.py index 45dc0a5b84..d93c602160 100644 --- a/tests/unit_tests/models/mistral4/test_mistral4_model.py +++ b/tests/unit_tests/models/mistral4/test_mistral4_model.py @@ -658,6 +658,18 @@ def test_forward_float_input_ids_as_embeds(self, text_config, backend, device): @_skip_no_hf_mistral3 class TestMistral3ForConditionalGeneration: + def test_checkpoint_tie_policy_ignores_outer_wrapper_default(self, multimodal_config, backend): + from nemo_automodel.components.checkpoint.utils import is_tied_word_embeddings + from nemo_automodel.components.models.mistral4.model import ( + Mistral3ForConditionalGeneration as OurMistral3ForCG, + ) + + model = OurMistral3ForCG(multimodal_config, backend=backend) + + assert multimodal_config.tie_word_embeddings is True + assert multimodal_config.text_config.tie_word_embeddings is False + assert is_tied_word_embeddings(model) is False + def test_init(self, multimodal_config, backend): from nemo_automodel.components.models.mistral4.model import ( Mistral3ForConditionalGeneration as OurMistral3ForCG, diff --git a/tests/unit_tests/models/qwen2/test_qwen2_tied_weights_cpu.py b/tests/unit_tests/models/qwen2/test_qwen2_tied_weights_cpu.py new file mode 100644 index 0000000000..d8a4ded226 --- /dev/null +++ b/tests/unit_tests/models/qwen2/test_qwen2_tied_weights_cpu.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tied/untied alias tests for Qwen2ForCausalLM (TieSupport.BOTH). + +Qwen2 ships both tied (0.5B-3B) and untied (7B/72B) checkpoints, so it declares +BOTH and honors the flag either way. These pin the explicit ``tie_weights()`` +override (added because HF's base machinery does not reliably tie this custom +model from the dict-shaped ``_tied_weights_keys`` under transformers v5). +""" + +from transformers import Qwen2Config + +from nemo_automodel.components.models.qwen2.model import Qwen2ForCausalLM + + +def _tiny_qwen2_config(tie_word_embeddings: bool) -> Qwen2Config: + return Qwen2Config( + vocab_size=32, + hidden_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=2, + num_key_value_heads=1, + max_position_embeddings=16, + tie_word_embeddings=tie_word_embeddings, + ) + + +def test_qwen2_ties_lm_head_when_config_requests_tied_embeddings(): + model = Qwen2ForCausalLM(_tiny_qwen2_config(tie_word_embeddings=True)) + assert model.lm_head.weight is model.model.embed_tokens.weight + + +def test_qwen2_leaves_lm_head_untied_when_config_requests_untied_embeddings(): + model = Qwen2ForCausalLM(_tiny_qwen2_config(tie_word_embeddings=False)) + assert model.lm_head.weight is not model.model.embed_tokens.weight diff --git a/tests/unit_tests/models/qwen3_5/test_qwen3_5_tied_weights_cpu.py b/tests/unit_tests/models/qwen3_5/test_qwen3_5_tied_weights_cpu.py new file mode 100644 index 0000000000..9ba9ff6417 --- /dev/null +++ b/tests/unit_tests/models/qwen3_5/test_qwen3_5_tied_weights_cpu.py @@ -0,0 +1,124 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tied/untied alias tests for the Qwen3.5 dense family (TieSupport.BOTH). + +Supported dense Qwen3.5 checkpoints are mixed by size (0.8B/2B/4B tied, +9B/27B + Qwen3.6-27B untied), so both classes declare BOTH. + +The VLM class is the important regression guard: its ``__init__`` replaces +``self.model.language_model`` with ``Qwen3_5DenseTextBackbone`` *after* HF's +``post_init`` tied ``lm_head`` to the original embedding, orphaning that alias. +Its ``tie_weights()`` re-ties to the active backbone embedding when tie=True. + +Runs on CPU (torch backends, no TE / DeepEP). +""" + +import torch.nn as nn +from transformers.models.qwen3_5.configuration_qwen3_5 import ( + Qwen3_5Config, + Qwen3_5TextConfig, + Qwen3_5VisionConfig, +) + +from nemo_automodel.components.models.common import BackendConfig +from nemo_automodel.components.models.qwen3_5.model import ( + Qwen3_5ForCausalLM, + Qwen3_5ForConditionalGeneration, +) + + +def _backend() -> BackendConfig: + """CPU-friendly backend: plain torch kernels, no fused RoPE.""" + return BackendConfig( + linear="torch", + attn="sdpa", + rms_norm="torch", + rope_fusion=False, + dispatcher="torch", + fake_balanced_gate=False, + enable_hf_state_dict_adapter=True, + ) + + +def _tiny_text_config(tie_word_embeddings: bool) -> Qwen3_5TextConfig: + return Qwen3_5TextConfig( + vocab_size=64, + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=2, + num_key_value_heads=2, + head_dim=8, + intermediate_size=32, + max_position_embeddings=16, + rms_norm_eps=1e-6, + pad_token_id=0, + layer_types=["full_attention"], + attn_implementation="eager", + torch_dtype="float32", + tie_word_embeddings=tie_word_embeddings, + ) + + +def _tiny_vlm_config(tie_word_embeddings: bool) -> Qwen3_5Config: + text_config = _tiny_text_config(tie_word_embeddings) + vision_config = Qwen3_5VisionConfig( + depth=1, + hidden_size=16, + intermediate_size=32, + num_heads=2, + patch_size=2, + spatial_merge_size=1, + temporal_patch_size=1, + out_hidden_size=16, + ) + return Qwen3_5Config( + architectures=["Qwen3_5ForConditionalGeneration"], + text_config=text_config.to_dict(), + vision_config=vision_config.to_dict(), + image_token_id=60, + video_token_id=61, + vision_start_token_id=62, + vision_end_token_id=63, + tie_word_embeddings=tie_word_embeddings, + ) + + +class TestQwen3_5CausalTieWeights: + def test_tied_shares_lm_head_storage(self): + model = Qwen3_5ForCausalLM(_tiny_text_config(tie_word_embeddings=True), backend=_backend()) + assert model.lm_head.weight is model.model.embed_tokens.weight + + def test_untied_has_separate_lm_head(self): + model = Qwen3_5ForCausalLM(_tiny_text_config(tie_word_embeddings=False), backend=_backend()) + assert model.lm_head.weight is not model.model.embed_tokens.weight + + +class TestQwen3_5ConditionalGenerationTieWeights: + def test_tied_reties_to_active_backbone_after_swap(self): + model = Qwen3_5ForConditionalGeneration(_tiny_vlm_config(tie_word_embeddings=True), backend=_backend()) + assert model.lm_head.weight is model.model.language_model.embed_tokens.weight + + def test_tie_weights_restores_post_swap_alias(self): + model = Qwen3_5ForConditionalGeneration(_tiny_vlm_config(tie_word_embeddings=True), backend=_backend()) + model.lm_head.weight = nn.Parameter(model.lm_head.weight.detach().clone()) + assert model.lm_head.weight is not model.model.language_model.embed_tokens.weight + + model.tie_weights() + + assert model.lm_head.weight is model.model.language_model.embed_tokens.weight + + def test_untied_has_separate_lm_head(self): + model = Qwen3_5ForConditionalGeneration(_tiny_vlm_config(tie_word_embeddings=False), backend=_backend()) + assert model.lm_head.weight is not model.model.language_model.embed_tokens.weight diff --git a/tests/unit_tests/models/qwen3_moe/test_qwen3_moe_tie_guard.py b/tests/unit_tests/models/qwen3_moe/test_qwen3_moe_tie_guard.py index d8b1be33dc..a534045431 100644 --- a/tests/unit_tests/models/qwen3_moe/test_qwen3_moe_tie_guard.py +++ b/tests/unit_tests/models/qwen3_moe/test_qwen3_moe_tie_guard.py @@ -17,7 +17,9 @@ The reject guard runs at the very start of ``__init__`` (before any device- or kernel-dependent construction), so this is CPU-safe even though the full qwen3_moe model build requires a GPU. qwen3_moe stands in for the whole -untied-default family wired through ``reject_unsupported_tied_word_embeddings``. +untied-default family, which declares ``tie_word_embeddings_support = +TieSupport.UNTIED_ONLY`` and is validated by +``reject_unsupported_tie_word_embeddings``. """ import pytest diff --git a/tests/unit_tests/utils/test_checkpoint_utils.py b/tests/unit_tests/utils/test_checkpoint_utils.py index a28d7ec2bb..1e16f4ccf4 100644 --- a/tests/unit_tests/utils/test_checkpoint_utils.py +++ b/tests/unit_tests/utils/test_checkpoint_utils.py @@ -18,6 +18,7 @@ import torch.nn as nn import nemo_automodel.components.checkpoint.utils as checkpoint_utils +import nemo_automodel.components.models.common.tie_word_embeddings as tie_utils def test_is_tied_word_embeddings_prefers_top_level_value(): @@ -87,6 +88,44 @@ def __init__(self) -> None: assert checkpoint_utils.is_tied_word_embeddings(model) is False +def test_is_tied_word_embeddings_uses_one_direction_policy_over_outer_config(): + """A fixed model policy wins over a misleading composite outer flag.""" + + class UntiedOnlyModel(nn.Module): + tie_word_embeddings_support = tie_utils.TieSupport.UNTIED_ONLY + + def __init__(self) -> None: + super().__init__() + self.config = SimpleNamespace( + tie_word_embeddings=True, + text_config=SimpleNamespace(tie_word_embeddings=False), + ) + + class TiedOnlyModel(nn.Module): + tie_word_embeddings_support = tie_utils.TieSupport.TIED_ONLY + + def __init__(self) -> None: + super().__init__() + self.config = SimpleNamespace(tie_word_embeddings=False) + + assert checkpoint_utils.is_tied_word_embeddings(UntiedOnlyModel()) is False + assert checkpoint_utils.is_tied_word_embeddings(TiedOnlyModel()) is True + + +def test_is_tied_word_embeddings_both_follows_outer_config(): + """A BOTH model still resolves its per-checkpoint top-level flag.""" + + class BothModel(nn.Module): + tie_word_embeddings_support = tie_utils.TieSupport.BOTH + + def __init__(self, tied: bool) -> None: + super().__init__() + self.config = SimpleNamespace(tie_word_embeddings=tied) + + assert checkpoint_utils.is_tied_word_embeddings(BothModel(tied=True)) is True + assert checkpoint_utils.is_tied_word_embeddings(BothModel(tied=False)) is False + + def test_is_tied_word_embeddings_qwen3_omni_moe_follows_top_level(): """Qwen3OmniMoeThinker reports its top-level config intent. @@ -112,8 +151,8 @@ def test_get_controlling_tie_word_embeddings_top_level_first(): cfg_top_false = SimpleNamespace( tie_word_embeddings=False, get_text_config=lambda: SimpleNamespace(tie_word_embeddings=True) ) - assert checkpoint_utils.get_controlling_tie_word_embeddings(cfg_top_true, "SomeForCausalLM") is True - assert checkpoint_utils.get_controlling_tie_word_embeddings(cfg_top_false, "SomeForCausalLM") is False + assert tie_utils.get_controlling_tie_word_embeddings(cfg_top_true) is True + assert tie_utils.get_controlling_tie_word_embeddings(cfg_top_false) is False def test_get_controlling_tie_word_embeddings_falls_back_to_text_config(): @@ -123,33 +162,45 @@ class _NoTopFlag: def get_text_config(self): return SimpleNamespace(tie_word_embeddings=True) - assert checkpoint_utils.get_controlling_tie_word_embeddings(_NoTopFlag(), "SomeForCausalLM") is True + assert tie_utils.get_controlling_tie_word_embeddings(_NoTopFlag()) is True -def test_reject_unsupported_tied_word_embeddings_raises_when_tied(): - """A separate-head model with tie_word_embeddings=True is rejected.""" +def _model_cls(name: str, support: tie_utils.TieSupport) -> type: + """Build a throwaway model class with the given name and TieSupport policy. + + The class name selects the resolver's composite/omni special-casing and the + ``tie_word_embeddings_support`` attribute drives the guard, matching how real + registered model classes declare their policy. + """ + return type(name, (), {"tie_word_embeddings_support": support}) + + +def test_reject_unsupported_tie_word_embeddings_untied_only_raises_when_tied(): + """An UNTIED_ONLY (separate-head) class with tie_word_embeddings=True is rejected.""" + cls = _model_cls("Qwen3MoeForCausalLM", tie_utils.TieSupport.UNTIED_ONLY) config = SimpleNamespace(tie_word_embeddings=True) with pytest.raises(NotImplementedError, match="does not support tie_word_embeddings=True"): - checkpoint_utils.reject_unsupported_tied_word_embeddings(config, "Qwen3MoeForCausalLM") + tie_utils.reject_unsupported_tie_word_embeddings(cls, config) -def test_reject_unsupported_tied_word_embeddings_noop_when_untied(): - """The default (untied) config passes the guard without raising.""" - config = SimpleNamespace(tie_word_embeddings=False) - checkpoint_utils.reject_unsupported_tied_word_embeddings(config, "Qwen3MoeForCausalLM") # no raise +def test_reject_unsupported_tie_word_embeddings_untied_only_noop_when_untied(): + """The default (untied) config passes an UNTIED_ONLY guard without raising.""" + cls = _model_cls("Qwen3MoeForCausalLM", tie_utils.TieSupport.UNTIED_ONLY) + tie_utils.reject_unsupported_tie_word_embeddings(cls, SimpleNamespace(tie_word_embeddings=False)) # no raise -def test_reject_unsupported_tied_word_embeddings_uses_top_level_for_composite(): - """Composite VLM/omni configs read the controlling top-level flag, not nested text_config.""" +def test_reject_unsupported_tie_word_embeddings_uses_top_level_for_composite(): + """Composite VLM configs read the controlling top-level flag, not nested text_config.""" + cls = _model_cls("Qwen3VLMoeForConditionalGeneration", tie_utils.TieSupport.UNTIED_ONLY) # top-level False (even with nested text True) -> not tied -> no raise untied = SimpleNamespace( tie_word_embeddings=False, get_text_config=lambda: SimpleNamespace(tie_word_embeddings=True) ) - checkpoint_utils.reject_unsupported_tied_word_embeddings(untied, "Qwen3VLMoeForConditionalGeneration") + tie_utils.reject_unsupported_tie_word_embeddings(cls, untied) # top-level True -> tied -> raise tied = SimpleNamespace(tie_word_embeddings=True, get_text_config=lambda: SimpleNamespace(tie_word_embeddings=False)) with pytest.raises(NotImplementedError): - checkpoint_utils.reject_unsupported_tied_word_embeddings(tied, "Qwen3VLMoeForConditionalGeneration") + tie_utils.reject_unsupported_tie_word_embeddings(cls, tied) def test_get_controlling_tie_word_embeddings_omni_wrapper_reads_thinker_config(): @@ -160,42 +211,73 @@ def test_get_controlling_tie_word_embeddings_omni_wrapper_reads_thinker_config() """ wrapper_tied = SimpleNamespace(thinker_config=SimpleNamespace(tie_word_embeddings=True)) wrapper_untied = SimpleNamespace(thinker_config=SimpleNamespace(tie_word_embeddings=False)) - for cls in ( - "Qwen2_5OmniThinkerForConditionalGeneration", - "Qwen3OmniMoeThinkerForConditionalGeneration", - ): - assert checkpoint_utils.get_controlling_tie_word_embeddings(wrapper_tied, cls) is True - assert checkpoint_utils.get_controlling_tie_word_embeddings(wrapper_untied, cls) is False + assert tie_utils.get_controlling_tie_word_embeddings(wrapper_tied) is True + assert tie_utils.get_controlling_tie_word_embeddings(wrapper_untied) is False # When the thinker config itself is passed (no nested thinker_config), read its own flag. direct = SimpleNamespace(tie_word_embeddings=True) - assert ( - checkpoint_utils.get_controlling_tie_word_embeddings(direct, "Qwen2_5OmniThinkerForConditionalGeneration") - is True - ) + assert tie_utils.get_controlling_tie_word_embeddings(direct) is True -def test_reject_unsupported_tied_word_embeddings_omni_wrapper_path(): +def test_reject_unsupported_tie_word_embeddings_omni_wrapper_path(): """The guard raises for a full Omni wrapper whose thinker_config requests tying.""" + tied_cls = _model_cls("Qwen2_5OmniThinkerForConditionalGeneration", tie_utils.TieSupport.UNTIED_ONLY) wrapper = SimpleNamespace(thinker_config=SimpleNamespace(tie_word_embeddings=True)) with pytest.raises(NotImplementedError): - checkpoint_utils.reject_unsupported_tied_word_embeddings(wrapper, "Qwen2_5OmniThinkerForConditionalGeneration") + tie_utils.reject_unsupported_tie_word_embeddings(tied_cls, wrapper) + untied_cls = _model_cls("Qwen3OmniMoeThinkerForConditionalGeneration", tie_utils.TieSupport.UNTIED_ONLY) wrapper_untied = SimpleNamespace(thinker_config=SimpleNamespace(tie_word_embeddings=False)) - checkpoint_utils.reject_unsupported_tied_word_embeddings( - wrapper_untied, "Qwen3OmniMoeThinkerForConditionalGeneration" - ) # no raise + tie_utils.reject_unsupported_tie_word_embeddings(untied_cls, wrapper_untied) # no raise -def test_reject_unsupported_untied_word_embeddings_raises_when_untied(): - """A tied-default model with tie_word_embeddings=False is rejected.""" +def test_reject_unsupported_tie_word_embeddings_tied_only_raises_when_untied(): + """A TIED_ONLY model with tie_word_embeddings=False is rejected.""" + cls = _model_cls("Gemma4ForConditionalGeneration", tie_utils.TieSupport.TIED_ONLY) config = SimpleNamespace(tie_word_embeddings=False) with pytest.raises(NotImplementedError, match="does not support tie_word_embeddings=False"): - checkpoint_utils.reject_unsupported_untied_word_embeddings(config, "Gemma4ForConditionalGeneration") + tie_utils.reject_unsupported_tie_word_embeddings(cls, config) -def test_reject_unsupported_untied_word_embeddings_noop_when_tied(): - """The default (tied) config passes the untie guard without raising.""" - config = SimpleNamespace(tie_word_embeddings=True) - checkpoint_utils.reject_unsupported_untied_word_embeddings(config, "Gemma4ForConditionalGeneration") # no raise +def test_reject_unsupported_tie_word_embeddings_tied_only_noop_when_tied(): + """The default (tied) config passes a TIED_ONLY guard without raising.""" + cls = _model_cls("Gemma4ForConditionalGeneration", tie_utils.TieSupport.TIED_ONLY) + tie_utils.reject_unsupported_tie_word_embeddings(cls, SimpleNamespace(tie_word_embeddings=True)) # no raise + + +def test_reject_unsupported_tie_word_embeddings_both_is_noop(): + """A BOTH class accepts either tying value without raising.""" + cls = _model_cls("LlamaForCausalLM", tie_utils.TieSupport.BOTH) + tie_utils.reject_unsupported_tie_word_embeddings(cls, SimpleNamespace(tie_word_embeddings=True)) # no raise + tie_utils.reject_unsupported_tie_word_embeddings(cls, SimpleNamespace(tie_word_embeddings=False)) # no raise + + +def test_reject_unsupported_tie_word_embeddings_defaults_to_both(): + """A class that does not declare a policy defaults to BOTH (guard is a no-op).""" + + class _Undeclared: + pass + + tie_utils.reject_unsupported_tie_word_embeddings(_Undeclared, SimpleNamespace(tie_word_embeddings=True)) + tie_utils.reject_unsupported_tie_word_embeddings(_Undeclared, SimpleNamespace(tie_word_embeddings=False)) + + +def test_reject_tie_word_embeddings_flip_raises_on_mismatch(): + """from_pretrained flip guard rejects a requested tie value differing from the checkpoint's.""" + tied = SimpleNamespace(tie_word_embeddings=True) + untied = SimpleNamespace(tie_word_embeddings=False) + # untied checkpoint, tied requested + with pytest.raises(NotImplementedError, match="flipping the flag is not supported"): + tie_utils.reject_tie_word_embeddings_flip(untied, tied, "LlamaForCausalLM") + # tied checkpoint, untied requested (both directions rejected) + with pytest.raises(NotImplementedError, match="flipping the flag is not supported"): + tie_utils.reject_tie_word_embeddings_flip(tied, untied, "LlamaForCausalLM") + + +def test_reject_tie_word_embeddings_flip_noop_when_matching(): + """No raise when the requested value matches the checkpoint's (either direction).""" + for tie in (True, False): + tie_utils.reject_tie_word_embeddings_flip( + SimpleNamespace(tie_word_embeddings=tie), SimpleNamespace(tie_word_embeddings=tie), "LlamaForCausalLM" + ) class _DraftLikeModel(nn.Module):