Add dots3-note Preview model support

docs/source/en/_toctree.yml

@@ -1307,6 +1307,8 @@
         title: DiffusionGemma
       - local: model_doc/donut
         title: Donut
+      - local: model_doc/dots3_note
+        title: Dots 3 Note Preview
       - local: model_doc/edgetam
         title: EdgeTAM
       - local: model_doc/edgetam_video

docs/source/en/model_doc/dots3_note.md

@@ -0,0 +1,142 @@
+<!--Copyright 2026 The Dots Studio team and the HuggingFace Inc. team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+⚠️ Note that this file is in Markdown but contains specific syntax for our doc-builder (similar to MDX) that may not be
+rendered properly in your Markdown viewer.
+
+-->
+*This model was contributed to Hugging Face Transformers on 2026-09-09.*
+
+# Dots 3 Note Preview
+
+Dots 3 Note Preview is a mixture-of-experts causal language model with native text, image, video, and audio inputs. It uses
+a shared vision encoder for images and videos and a Whisper-style audio encoder. Both encoders project their outputs
+into the language model's hidden space before autoregressive text generation.
+
+Dots 3 Note Preview checkpoints are available with BF16 weights or with fine-grained FP8 language-model weights and BF16
+vision, audio, and language-model-head weights. Both formats load through the standard Transformers APIs.
+
+## Usage
+
+```python
+from transformers import AutoModelForMultimodalLM, AutoProcessor
+
+
+model_id = "dots-studio/dots3-note-prev"
+processor = AutoProcessor.from_pretrained(model_id)
+model = AutoModelForMultimodalLM.from_pretrained(model_id, device_map="auto")
+
+conversation = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/cats.png"},
+            {"type": "text", "text": "Describe this image."},
+        ],
+    }
+]
+inputs = processor.apply_chat_template(
+    conversation,
+    add_generation_prompt=True,
+    tokenize=True,
+    return_dict=True,
+    return_tensors="pt",
+).to(model.device)
+
+output_ids = model.generate(**inputs, max_new_tokens=128)
+print(processor.batch_decode(output_ids, skip_special_tokens=True)[0])
+```
+
+Use the same processing and generation calls with any of these user messages:
+
+```python
+# Text
+conversation = [{"role": "user", "content": [{"type": "text", "text": "Briefly introduce yourself."}]}]
+
+# Audio: replace the path with a local recording.
+conversation = [{"role": "user", "content": [
+    {"type": "audio", "path": "speech.wav"},
+    {"type": "text", "text": "Transcribe this recording."},
+]}]
+
+# Native video, including its audio track when present.
+conversation = [{"role": "user", "content": [
+    {"type": "video", "path": "concert.mp4"},
+    {"type": "text", "text": "Describe what you see and hear."},
+]}]
+```
+
+`Dots3NoteModel` combines the text, vision and audio encoders. `Dots3NoteForConditionalGeneration` adds the
+language-model head and generation interface. `Dots3NoteTextForCausalLM` provides the text-only variant;
+`Dots3NoteForCausalLM` remains a compatibility name for the original multimodal checkpoints.
+
+## Dots3NoteConfig
+
+[[autodoc]] Dots3NoteConfig
+
+## Dots3NoteVisionConfig
+
+[[autodoc]] Dots3NoteVisionConfig
+
+## Dots3NoteAudioConfig
+
+[[autodoc]] Dots3NoteAudioConfig
+
+## Dots3NoteForCausalLM
+
+[[autodoc]] Dots3NoteForCausalLM
+    - forward
+
+## Dots3NoteModel
+
+[[autodoc]] Dots3NoteModel
+    - forward
+
+## Dots3NoteForConditionalGeneration
+
+[[autodoc]] Dots3NoteForConditionalGeneration
+
+## Dots3NoteTextModel
+
+[[autodoc]] Dots3NoteTextModel
+    - forward
+
+## Dots3NoteTextForCausalLM
+
+[[autodoc]] Dots3NoteTextForCausalLM
+    - forward
+
+## Dots3NoteVisionModel
+
+[[autodoc]] Dots3NoteVisionModel
+    - forward
+
+## Dots3NoteAudioModel
+
+[[autodoc]] Dots3NoteAudioModel
+    - forward
+
+## Dots3NoteProcessor
+
+[[autodoc]] Dots3NoteProcessor
+    - __call__
+
+## Dots3NoteImageProcessor
+
+[[autodoc]] Dots3NoteImageProcessor
+
+## Dots3NoteVideoProcessor
+
+[[autodoc]] Dots3NoteVideoProcessor
+
+## Dots3NoteFeatureExtractor
+
+[[autodoc]] Dots3NoteFeatureExtractor

src/transformers/conversion_mapping.py

@@ -1048,6 +1048,56 @@ def _build_checkpoint_conversion_mapping():
                 operations=[MergeModulelist(dim=0)],
             ),
         ],
+        "dots3_note_audio_encoder": [
+            WeightRenaming(r"dots_encoder\.speech_encoder\.", "speech_encoder."),
+            WeightRenaming(r"speech_encoder\.(conv2d[123]|conv_out)\.", r"speech_encoder.conv_stem.\1."),
+            WeightRenaming(r"(speech_encoder\.layers\.\d+\.self_attn)\.out_proj\.", r"\1.o_proj."),
+            WeightRenaming(r"\.self_attn_layer_norm\.", ".input_layernorm."),
+            WeightRenaming(r"\.final_layer_norm\.", ".post_attention_layernorm."),
+            WeightRenaming(r"(speech_encoder\.layers\.\d+)\.fc1\.", r"\1.mlp.gate_up_proj."),
+            WeightRenaming(r"(speech_encoder\.layers\.\d+)\.fc2\.", r"\1.mlp.down_proj."),
+            WeightRenaming(r"audio_adapter\.proj\.0\.", "audio_adapter.norm."),
+            WeightRenaming(r"audio_adapter\.proj\.1\.", "audio_adapter.fc1."),
+            WeightRenaming(r"audio_adapter\.proj\.3\.", "audio_adapter.fc2."),
+        ],
+        "dots3_note_vision_encoder": [
+            WeightRenaming(r"(blocks\.\d+)\.norm_1\.", r"\1.norm1."),
+            WeightRenaming(r"(blocks\.\d+)\.norm_2\.", r"\1.norm2."),
+            WeightRenaming(r"\.fc1\.", ".gate_proj."),
+            WeightRenaming(r"\.fc2\.", ".down_proj."),
+            WeightRenaming(r"\.fc3\.", ".up_proj."),
+        ],
+        "Dots3NoteModel": [
+            WeightRenaming(r"^(embed_tokens|layers|norm)\.", r"language_model.\1."),
+        ],
+        "Dots3NoteTextModel": [
+            WeightConverter(
+                source_patterns=[
+                    "mlp.experts.*.gate_proj.weight$",
+                    "mlp.experts.*.up_proj.weight$",
+                ],
+                target_patterns="mlp.experts.gate_up_proj",
+                operations=[MergeModulelist(dim=0), Concatenate(dim=1)],
+            ),
+            WeightConverter(
+                source_patterns="mlp.experts.*.down_proj.weight$",
+                target_patterns="mlp.experts.down_proj",
+                operations=[MergeModulelist(dim=0)],
+            ),
+            WeightConverter(
+                source_patterns=[
+                    "mlp.experts.*.gate_proj.weight_scale_inv$",
+                    "mlp.experts.*.up_proj.weight_scale_inv$",
+                ],
+                target_patterns="mlp.experts.gate_up_proj_scale_inv",
+                operations=[MergeModulelist(dim=0), Concatenate(dim=1)],
+            ),
+            WeightConverter(
+                source_patterns="mlp.experts.*.down_proj.weight_scale_inv$",
+                target_patterns="mlp.experts.down_proj_scale_inv",
+                operations=[MergeModulelist(dim=0)],
+            ),
+        ],
         "qwen3_vl_moe": [
             WeightConverter(
                 source_patterns="mlp.experts.gate_up_proj",

src/transformers/integrations/finegrained_fp8.py

@@ -934,22 +934,31 @@ def _resolve_block_size(self, value: torch.Tensor) -> tuple[int, int]:
         return tuple(block_size)
 
     def _quantize_one(self, key: str, value: torch.Tensor) -> dict[str, torch.Tensor]:
-        # Pass through tensors that aren't tileable (1D norms / biases, or shapes
-        # that don't divide cleanly by the configured block) — they were never
-        # FP8-quantized on the load side, so the reverse op shouldn't touch them.
+        # Norms and biases are not block-quantized.
         if value.ndim < 2:
             return {key: value}
         block_m, block_n = self._resolve_block_size(value)
         rows, cols = value.shape[-2], value.shape[-1]
-        if rows % block_m != 0 or cols % block_n != 0:
+        has_partial_block = rows % block_m != 0 or cols % block_n != 0
+        requantizing_pre_quantized_checkpoint = getattr(self.hf_quantizer, "pre_quantized", False) and getattr(
+            self.hf_quantizer.quantization_config, "dequantize", False
+        )
+        # Keep the existing on-the-fly quantization behavior for odd-shaped linears:
+        # they stay in full precision. Partial blocks are only required when reversing
+        # dequantization while saving a checkpoint that was already FP8-quantized.
+        if has_partial_block and not requantizing_pre_quantized_checkpoint:
             return {key: value}
 
         # Leading dims can be empty (2D) or include num_experts/... (3D+)
         leading_shape = value.shape[:-2]
-        rows_tiles = rows // block_m
-        cols_tiles = cols // block_n
+        rows_tiles = _cdiv(rows, block_m)
+        cols_tiles = _cdiv(cols, block_n)
+        padded_rows = rows_tiles * block_m
+        padded_cols = cols_tiles * block_n
         original_shape = value.shape
         value_fp32 = value.to(torch.float32)
+        if padded_rows != rows or padded_cols != cols:
+            value_fp32 = F.pad(value_fp32, (0, padded_cols - cols, 0, padded_rows - rows))
         # Reshape to (..., rows_tiles, block_m, cols_tiles, block_n)
         reshaped = value_fp32.reshape(*leading_shape, rows_tiles, block_m, cols_tiles, block_n)
         # Per-tile max-abs over the block dims (block_m at -3, block_n at -1)
@@ -969,6 +978,7 @@ def _quantize_one(self, key: str, value: torch.Tensor) -> dict[str, torch.Tensor
         scales_broadcast = scales.unsqueeze(-1).unsqueeze(-3)  # (..., rows_tiles, 1, cols_tiles, 1)
         scaled = reshaped * scales_broadcast
         quantized = torch.clamp(scaled, min=_FP8_MIN, max=_FP8_MAX).to(_FP8_DTYPE)
+        quantized = quantized.reshape(*leading_shape, padded_rows, padded_cols)[..., :rows, :cols].contiguous()
         quantized = quantized.reshape(original_shape)
         scale_key = key.rsplit(".", 1)[0] + ".weight_scale_inv" if key.endswith(".weight") else key + "_scale_inv"
         return {key: quantized, scale_key: inv_scales}
@@ -1041,25 +1051,38 @@ def _dequantize_one(
         # FP4 path: int8 / float4_e2m1fn_x2 stores two nibbles per byte. Unpack to fp32
         # first so the rest of the routine sees a normal (rows, cols) float matrix.
         fp4_dtype = getattr(torch, "float4_e2m1fn_x2", None)
-        if quantized.dtype == torch.int8 or (fp4_dtype is not None and quantized.dtype == fp4_dtype):
+        is_fp4 = quantized.dtype == torch.int8 or (fp4_dtype is not None and quantized.dtype == fp4_dtype)
+        if is_fp4:
             quantized_fp32 = self._unpack_fp4(quantized)
         else:
             quantized_fp32 = quantized.to(torch.float32)
         rows, cols = quantized_fp32.shape[-2:]
-        # Derive block size from the scale grid rather than the global config: MoE experts
-        # ship MXFP4 with a ``[1, 32]`` block, dense linears ship FP8 with ``[128, 128]``,
-        # and the same dequant has to handle both within one checkpoint.
         try:
             scale_rows, scale_cols = scales.shape[-2:]
         except Exception:
             # scale can be a single tensor in extreme cases where it was not wrapped properly but is [1,0].
             scale_rows, scale_cols = 1, 1
-        if rows % scale_rows or cols % scale_cols:
-            raise ValueError(
-                f"Weight shape ({rows}, {cols}) not divisible by scale grid ({scale_rows}, {scale_cols})."
-            )
-        block_m = rows // scale_rows
-        block_n = cols // scale_cols
+        quantization_config = self.hf_quantizer.quantization_config
+        block_size = (
+            quantization_config.get("weight_block_size")
+            if isinstance(quantization_config, dict)
+            else getattr(quantization_config, "weight_block_size", None)
+        )
+        # Use configured FP8 blocks when their ceil-divided grid matches, including partial blocks.
+        # FP4 and legacy layouts retain the scale-grid-derived behavior.
+        if (
+            not is_fp4
+            and block_size is not None
+            and (scale_rows, scale_cols) == (_cdiv(rows, block_size[0]), _cdiv(cols, block_size[1]))
+        ):
+            block_m, block_n = block_size
+        else:
+            if rows % scale_rows or cols % scale_cols:
+                raise ValueError(
+                    f"Weight shape ({rows}, {cols}) not divisible by scale grid ({scale_rows}, {scale_cols})."
+                )
+            block_m = rows // scale_rows
+            block_n = cols // scale_cols
         # ``ue8m0`` (``float8_e8m0fnu``) scales have no CUDA ``mul`` kernel, and casting
         # the FP8 weight to that dtype loses precision. Promote both sides to fp32 for
         # the math; prefer the destination parameter's dtype when known so eager modules
@@ -1076,9 +1099,14 @@ def _dequantize_one(
         else:
             s_fp32 = scales.to(torch.float32)
         original_shape = quantized_fp32.shape
+        padded_rows = scale_rows * block_m
+        padded_cols = scale_cols * block_n
+        if padded_rows != rows or padded_cols != cols:
+            quantized_fp32 = F.pad(quantized_fp32, (0, padded_cols - cols, 0, padded_rows - rows))
         q = quantized_fp32.reshape(-1, scale_rows, block_m, scale_cols, block_n)
         s = s_fp32.reshape(-1, scale_rows, scale_cols).unsqueeze(-1).unsqueeze(2)
-        return (q * s).to(output_dtype).reshape(original_shape)
+        dequantized = (q * s).reshape(*original_shape[:-2], padded_rows, padded_cols)[..., :rows, :cols]
+        return dequantized.to(output_dtype).contiguous()
 
     def _get_target_dtype(self, model: torch.nn.Module | None, full_layer_name: str | None) -> torch.dtype | None:
         if model is None or full_layer_name is None:

src/transformers/models/__init__.py

@@ -129,6 +129,7 @@
     from .doge import *
     from .donut import *
     from .dots1 import *
+    from .dots3_note import *
     from .dpr import *
     from .dpt import *
     from .edgetam import *

src/transformers/models/auto/auto_mappings.py

@@ -164,6 +164,9 @@
         ("doge", "DogeConfig"),
         ("donut-swin", "DonutSwinConfig"),
         ("dots1", "Dots1Config"),
+        ("dots3_note", "Dots3NoteConfig"),
+        ("dots3_note_audio_encoder", "Dots3NoteAudioConfig"),
+        ("dots3_note_vision_encoder", "Dots3NoteVisionConfig"),
         ("dpr", "DPRConfig"),
         ("dpt", "DPTConfig"),
         ("edgetam", "EdgeTamConfig"),
@@ -790,6 +793,8 @@
         ("dia_encoder", "dia"),
         ("diffusion_gemma_text", "diffusion_gemma"),
         ("donut-swin", "donut"),
+        ("dots3_note_audio_encoder", "dots3_note"),
+        ("dots3_note_vision_encoder", "dots3_note"),
         ("edgetam_vision_model", "edgetam"),
         ("emu3_text_model", "emu3"),
         ("emu3_vqgan", "emu3"),
@@ -1033,6 +1038,7 @@
         ("cohere_asr", "CohereAsrFeatureExtractor"),
         ("dac", "DacFeatureExtractor"),
         ("dia", "DiaFeatureExtractor"),
+        ("dots3_note", "Dots3NoteFeatureExtractor"),
         ("encodec", "EncodecFeatureExtractor"),
         ("gemma3n", "Gemma3nAudioFeatureExtractor"),
         ("gemma4", "Gemma4AudioFeatureExtractor"),
@@ -1089,6 +1095,7 @@
         ("deepseek_vl", "DeepseekVLProcessor"),
         ("deepseek_vl_hybrid", "DeepseekVLHybridProcessor"),
         ("dia", "DiaProcessor"),
+        ("dots3_note", "Dots3NoteProcessor"),
         ("emu3", "Emu3Processor"),
         ("ernie4_5_vl_moe", "Ernie4_5_VLMoeProcessor"),
         ("evolla", "EvollaProcessor"),

src/transformers/models/auto/image_processing_auto.py

@@ -87,6 +87,7 @@
             ("dinat", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),
             ("dinov2", {"torchvision": "BitImageProcessor", "pil": "BitImageProcessorPil"}),
             ("donut-swin", {"torchvision": "DonutImageProcessor", "pil": "DonutImageProcessorPil"}),
+            ("dots3_note", {"pil": "Dots3NoteImageProcessor"}),
             ("edgetam", {"torchvision": "Sam2ImageProcessor"}),
             ("emu3", {"pil": "Emu3ImageProcessor"}),
             ("eomt_dinov3", {"torchvision": "EomtImageProcessor", "pil": "EomtImageProcessorPil"}),

src/transformers/models/auto/modeling_auto.py

@@ -146,6 +146,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("doge", "DogeModel"),
         ("donut-swin", "DonutSwinModel"),
         ("dots1", "Dots1Model"),
+        ("dots3_note", "Dots3NoteModel"),
         ("dpr", "DPRQuestionEncoder"),
         ("dpt", "DPTModel"),
         ("edgetam", "EdgeTamModel"),
@@ -734,6 +735,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("diffllama", "DiffLlamaForCausalLM"),
         ("doge", "DogeForCausalLM"),
         ("dots1", "Dots1ForCausalLM"),
+        ("dots3_note", "Dots3NoteForCausalLM"),
         ("electra", "ElectraForCausalLM"),
         ("emu3", "Emu3ForCausalLM"),
         ("ernie", "ErnieForCausalLM"),
@@ -1100,6 +1102,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("deepseek_vl", "DeepseekVLForConditionalGeneration"),
         ("deepseek_vl_hybrid", "DeepseekVLHybridForConditionalGeneration"),
         ("diffusion_gemma", "DiffusionGemmaForBlockDiffusion"),
+        ("dots3_note", "Dots3NoteForConditionalGeneration"),
         ("emu3", "Emu3ForConditionalGeneration"),
         ("ernie4_5_vl_moe", "Ernie4_5_VLMoeForConditionalGeneration"),
         ("evolla", "EvollaForProteinText2Text"),