Add dots3-note Preview model support

docs/source/en/_toctree.yml

@@ -1301,6 +1301,8 @@
         title: DiffusionGemma
       - local: model_doc/donut
         title: Donut
+      - local: model_doc/dots3_note
+        title: Dots 3 Note Preview
       - local: model_doc/edgetam
         title: EdgeTAM
       - local: model_doc/edgetam_video

docs/source/en/model_doc/dots3_note.md

@@ -0,0 +1,123 @@
+<!--Copyright 2026 The Dots Studio team and the HuggingFace Inc. team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+⚠️ Note that this file is in Markdown but contains specific syntax for our doc-builder (similar to MDX) that may not be
+rendered properly in your Markdown viewer.
+
+-->
+*This model was contributed to Hugging Face Transformers on 2026-09-04.*
+
+# Dots 3 Note Preview
+
+Dots 3 Note Preview is a mixture-of-experts causal language model with native text, image, video, and audio inputs. It uses
+a shared vision encoder for images and videos and a Whisper-style audio encoder. Both encoders project their outputs
+into the language model's hidden space before autoregressive text generation.
+
+Dots 3 Note Preview checkpoints are available with BF16 weights or with fine-grained FP8 language-model weights and BF16
+vision, audio, and language-model-head weights. Both formats load through the standard Transformers APIs.
+
+> [!NOTE]
+> Dots 3 Note Preview support is inference-only. In particular, the DSA indexer's discrete FP8 top-k selection runs under
+> `torch.no_grad()`, so its parameters do not receive gradients and training or fine-tuning is not supported.
+
+> [!NOTE]
+> Loading encoded audio or video sources requires the optional `torchcodec` dependency. Native video preprocessing
+> follows the training-time sampling pipeline and expands each video into timestamped image blocks with interleaved
+> audio blocks. The `<|video_pad|>` marker is only an external prompt placeholder and is removed before tokenization;
+> it is never passed to the model. Decoded frame arrays are supported as visual-only inputs because their original
+> audio track is absent.
+
+```python
+from transformers import AutoModelForMultimodalLM, AutoProcessor
+
+
+model_id = "dots-studio/dots3-note-prev"
+processor = AutoProcessor.from_pretrained(model_id)
+model = AutoModelForMultimodalLM.from_pretrained(model_id, device_map="auto")
+
+conversation = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/cats.png"},
+            {"type": "text", "text": "Describe this image."},
+        ],
+    }
+]
+inputs = processor.apply_chat_template(
+    conversation,
+    add_generation_prompt=True,
+    tokenize=True,
+    return_dict=True,
+    return_tensors="pt",
+).to(model.device)
+
+output_ids = model.generate(**inputs, max_new_tokens=128)
+print(processor.batch_decode(output_ids, skip_special_tokens=True)[0])
+```
+
+## Dots3NoteConfig
+
+[[autodoc]] Dots3NoteConfig
+
+## Dots3NoteVisionConfig
+
+[[autodoc]] Dots3NoteVisionConfig
+
+## Dots3NoteAudioConfig
+
+[[autodoc]] Dots3NoteAudioConfig
+
+## Dots3NoteForCausalLM
+
+[[autodoc]] Dots3NoteForCausalLM
+    - forward
+
+## Dots3NoteForConditionalGeneration
+
+[[autodoc]] Dots3NoteForConditionalGeneration
+
+## Dots3NoteTextModel
+
+[[autodoc]] Dots3NoteTextModel
+    - forward
+
+## Dots3NoteTextForCausalLM
+
+[[autodoc]] Dots3NoteTextForCausalLM
+    - forward
+
+## Dots3NoteVisionModel
+
+[[autodoc]] Dots3NoteVisionModel
+    - forward
+
+## Dots3NoteAudioModel
+
+[[autodoc]] Dots3NoteAudioModel
+    - forward
+
+## Dots3NoteProcessor
+
+[[autodoc]] Dots3NoteProcessor
+    - __call__
+
+## Dots3NoteImageProcessor
+
+[[autodoc]] Dots3NoteImageProcessor
+
+## Dots3NoteVideoProcessor
+
+[[autodoc]] Dots3NoteVideoProcessor
+
+## Dots3NoteFeatureExtractor
+
+[[autodoc]] Dots3NoteFeatureExtractor

src/transformers/conversion_mapping.py

@@ -1031,6 +1031,34 @@ def _build_checkpoint_conversion_mapping():
                 operations=[MergeModulelist(dim=0)],
             ),
         ],
+        "dots3_note": [
+            WeightConverter(
+                source_patterns=[
+                    "mlp.experts.*.gate_proj.weight$",
+                    "mlp.experts.*.up_proj.weight$",
+                ],
+                target_patterns="mlp.experts.gate_up_proj",
+                operations=[MergeModulelist(dim=0), Concatenate(dim=1)],
+            ),
+            WeightConverter(
+                source_patterns="mlp.experts.*.down_proj.weight$",
+                target_patterns="mlp.experts.down_proj",
+                operations=[MergeModulelist(dim=0)],
+            ),
+            WeightConverter(
+                source_patterns=[
+                    "mlp.experts.*.gate_proj.weight_scale_inv$",
+                    "mlp.experts.*.up_proj.weight_scale_inv$",
+                ],
+                target_patterns="mlp.experts.gate_up_proj_scale_inv",
+                operations=[MergeModulelist(dim=0), Concatenate(dim=1)],
+            ),
+            WeightConverter(
+                source_patterns="mlp.experts.*.down_proj.weight_scale_inv$",
+                target_patterns="mlp.experts.down_proj_scale_inv",
+                operations=[MergeModulelist(dim=0)],
+            ),
+        ],
         "qwen3_vl_moe": [
             WeightConverter(
                 source_patterns="mlp.experts.gate_up_proj",

src/transformers/integrations/finegrained_fp8.py

@@ -914,6 +914,19 @@ def replace_with_fp8_linear(
     return model
 
 
+def _resolve_weight_block_size(hf_quantizer, value: torch.Tensor) -> tuple[int, int]:
+    block_size = None
+    quantization_config = hf_quantizer.quantization_config
+    if quantization_config is not None:
+        if isinstance(quantization_config, dict):
+            block_size = quantization_config.get("weight_block_size")
+        else:
+            block_size = getattr(quantization_config, "weight_block_size", None)
+    if block_size is None:
+        block_size = value.shape[-2:]
+    return tuple(block_size)
+
+
 class Fp8Quantize(ConversionOps):
     """
     A quantization operation that creates two tensors, weight and scale out of a weight.
@@ -923,33 +936,35 @@ def __init__(self, hf_quantizer):
         self.hf_quantizer = hf_quantizer
 
     def _resolve_block_size(self, value: torch.Tensor) -> tuple[int, int]:
-        block_size = None
-        if self.hf_quantizer.quantization_config is not None:
-            if isinstance(self.hf_quantizer.quantization_config, dict):
-                block_size = self.hf_quantizer.quantization_config.get("weight_block_size")
-            else:
-                block_size = getattr(self.hf_quantizer.quantization_config, "weight_block_size", None)
-        if block_size is None:
-            block_size = (value.shape[-2], value.shape[-1])
-        return tuple(block_size)
+        return _resolve_weight_block_size(self.hf_quantizer, value)
 
     def _quantize_one(self, key: str, value: torch.Tensor) -> dict[str, torch.Tensor]:
-        # Pass through tensors that aren't tileable (1D norms / biases, or shapes
-        # that don't divide cleanly by the configured block) — they were never
-        # FP8-quantized on the load side, so the reverse op shouldn't touch them.
+        # Norms and biases are not block-quantized.
         if value.ndim < 2:
             return {key: value}
         block_m, block_n = self._resolve_block_size(value)
         rows, cols = value.shape[-2], value.shape[-1]
-        if rows % block_m != 0 or cols % block_n != 0:
+        has_partial_block = rows % block_m != 0 or cols % block_n != 0
+        quantization_config = self.hf_quantizer.quantization_config
+        requantizing_pre_quantized_checkpoint = getattr(self.hf_quantizer, "pre_quantized", False) and getattr(
+            quantization_config, "dequantize", False
+        )
+        # Keep the existing on-the-fly quantization behavior for odd-shaped linears:
+        # they stay in full precision. Partial blocks are only required when reversing
+        # dequantization while saving a checkpoint that was already FP8-quantized.
+        if has_partial_block and not requantizing_pre_quantized_checkpoint:
             return {key: value}
 
         # Leading dims can be empty (2D) or include num_experts/... (3D+)
         leading_shape = value.shape[:-2]
-        rows_tiles = rows // block_m
-        cols_tiles = cols // block_n
+        rows_tiles = _cdiv(rows, block_m)
+        cols_tiles = _cdiv(cols, block_n)
+        padded_rows = rows_tiles * block_m
+        padded_cols = cols_tiles * block_n
         original_shape = value.shape
         value_fp32 = value.to(torch.float32)
+        if padded_rows != rows or padded_cols != cols:
+            value_fp32 = F.pad(value_fp32, (0, padded_cols - cols, 0, padded_rows - rows))
         # Reshape to (..., rows_tiles, block_m, cols_tiles, block_n)
         reshaped = value_fp32.reshape(*leading_shape, rows_tiles, block_m, cols_tiles, block_n)
         # Per-tile max-abs over the block dims (block_m at -3, block_n at -1)
@@ -969,6 +984,7 @@ def _quantize_one(self, key: str, value: torch.Tensor) -> dict[str, torch.Tensor
         scales_broadcast = scales.unsqueeze(-1).unsqueeze(-3)  # (..., rows_tiles, 1, cols_tiles, 1)
         scaled = reshaped * scales_broadcast
         quantized = torch.clamp(scaled, min=_FP8_MIN, max=_FP8_MAX).to(_FP8_DTYPE)
+        quantized = quantized.reshape(*leading_shape, padded_rows, padded_cols)[..., :rows, :cols].contiguous()
         quantized = quantized.reshape(original_shape)
         scale_key = key.rsplit(".", 1)[0] + ".weight_scale_inv" if key.endswith(".weight") else key + "_scale_inv"
         return {key: quantized, scale_key: inv_scales}
@@ -1041,25 +1057,43 @@ def _dequantize_one(
         # FP4 path: int8 / float4_e2m1fn_x2 stores two nibbles per byte. Unpack to fp32
         # first so the rest of the routine sees a normal (rows, cols) float matrix.
         fp4_dtype = getattr(torch, "float4_e2m1fn_x2", None)
-        if quantized.dtype == torch.int8 or (fp4_dtype is not None and quantized.dtype == fp4_dtype):
+        is_fp4 = quantized.dtype == torch.int8 or (fp4_dtype is not None and quantized.dtype == fp4_dtype)
+        if is_fp4:
             quantized_fp32 = self._unpack_fp4(quantized)
         else:
             quantized_fp32 = quantized.to(torch.float32)
         rows, cols = quantized_fp32.shape[-2:]
-        # Derive block size from the scale grid rather than the global config: MoE experts
-        # ship MXFP4 with a ``[1, 32]`` block, dense linears ship FP8 with ``[128, 128]``,
-        # and the same dequant has to handle both within one checkpoint.
         try:
             scale_rows, scale_cols = scales.shape[-2:]
         except Exception:
             # scale can be a single tensor in extreme cases where it was not wrapped properly but is [1,0].
             scale_rows, scale_cols = 1, 1
-        if rows % scale_rows or cols % scale_cols:
-            raise ValueError(
-                f"Weight shape ({rows}, {cols}) not divisible by scale grid ({scale_rows}, {scale_cols})."
-            )
-        block_m = rows // scale_rows
-        block_n = cols // scale_cols
+        if is_fp4:
+            # MXFP4 experts use a different block layout and keep the existing
+            # scale-grid-derived behavior.
+            if rows % scale_rows or cols % scale_cols:
+                raise ValueError(
+                    f"Weight shape ({rows}, {cols}) not divisible by scale grid ({scale_rows}, {scale_cols})."
+                )
+            block_m = rows // scale_rows
+            block_n = cols // scale_cols
+        else:
+            # FP8 checkpoints encode the block size in their quantization config. The
+            # scale grid uses ceil division, so its last row/column may describe a
+            # partial weight block.
+            block_m, block_n = _resolve_weight_block_size(self.hf_quantizer, quantized_fp32)
+            expected_scale_shape = (_cdiv(rows, block_m), _cdiv(cols, block_n))
+            if (scale_rows, scale_cols) != expected_scale_shape:
+                # Preserve the legacy path for existing checkpoints whose scale grid
+                # is valid but does not match the global block-size metadata.
+                if rows % scale_rows == 0 and cols % scale_cols == 0:
+                    block_m = rows // scale_rows
+                    block_n = cols // scale_cols
+                else:
+                    raise ValueError(
+                        f"Weight shape ({rows}, {cols}) with block size ({block_m}, {block_n}) expects scale grid "
+                        f"{expected_scale_shape}, but got ({scale_rows}, {scale_cols})."
+                    )
         # ``ue8m0`` (``float8_e8m0fnu``) scales have no CUDA ``mul`` kernel, and casting
         # the FP8 weight to that dtype loses precision. Promote both sides to fp32 for
         # the math; prefer the destination parameter's dtype when known so eager modules
@@ -1076,9 +1110,14 @@ def _dequantize_one(
         else:
             s_fp32 = scales.to(torch.float32)
         original_shape = quantized_fp32.shape
+        padded_rows = scale_rows * block_m
+        padded_cols = scale_cols * block_n
+        if padded_rows != rows or padded_cols != cols:
+            quantized_fp32 = F.pad(quantized_fp32, (0, padded_cols - cols, 0, padded_rows - rows))
         q = quantized_fp32.reshape(-1, scale_rows, block_m, scale_cols, block_n)
         s = s_fp32.reshape(-1, scale_rows, scale_cols).unsqueeze(-1).unsqueeze(2)
-        return (q * s).to(output_dtype).reshape(original_shape)
+        dequantized = (q * s).reshape(*original_shape[:-2], padded_rows, padded_cols)[..., :rows, :cols]
+        return dequantized.to(output_dtype).contiguous()
 
     def _get_target_dtype(self, model: torch.nn.Module | None, full_layer_name: str | None) -> torch.dtype | None:
         if model is None or full_layer_name is None:

src/transformers/integrations/moe.py

@@ -274,6 +274,9 @@ def _can_use_grouped_mm(input: torch.Tensor, weight: torch.Tensor, offs: torch.T
     # accept_dev=True is necessary for "+cpu"/"+xpu" etc.
     if (
         (is_torchdynamo_compiling() and weight.dtype != torch.bfloat16)
+        or weight.device.type == "cuda"
+        and is_torch_less_or_equal("2.8.0", accept_dev=True)
+        and weight.dtype != torch.bfloat16
         or weight.device.type == "cpu"
         and is_torch_less_or_equal("2.10.0", accept_dev=True)
         and (weight.data_ptr() % 16 != 0 or input.data_ptr() % 16 != 0)
@@ -282,9 +285,10 @@ def _can_use_grouped_mm(input: torch.Tensor, weight: torch.Tensor, offs: torch.T
     ):
         # We cannot use torch.grouped_mm and have to fall back when:
         # 1. torch.grouped_mm is not supported in torch.compile / inductor with dtypes other than bf16
-        # 2. on CPU with torch <= 2.10, the kernel requires 16 bytes alignment, which is not guaranteed for
+        # 2. torch._grouped_mm on CUDA with torch <= 2.8 only supports bf16
+        # 3. on CPU with torch <= 2.10, the kernel requires 16 bytes alignment, which is not guaranteed for
         #    tensors loaded using memmap (e.g. safetensors lazy loading)
-        # 3. on CPU with torch <= 2.8, torch._grouped_mm has no CPU kernel at all (raises NotImplementedError)
+        # 4. on CPU with torch <= 2.8, torch._grouped_mm has no CPU kernel at all (raises NotImplementedError)
         #    issue: https://github.com/pytorch/pytorch/issues/172440
         return False
 

src/transformers/models/auto/image_processing_auto.py

@@ -87,6 +87,7 @@
             ("dinat", {"torchvision": "ViTImageProcessor", "pil": "ViTImageProcessorPil"}),
             ("dinov2", {"torchvision": "BitImageProcessor", "pil": "BitImageProcessorPil"}),
             ("donut-swin", {"torchvision": "DonutImageProcessor", "pil": "DonutImageProcessorPil"}),
+            ("dots3_note", {"pil": "Dots3NoteImageProcessor"}),
             ("edgetam", {"torchvision": "Sam2ImageProcessor"}),
             ("emu3", {"pil": "Emu3ImageProcessor"}),
             ("eomt_dinov3", {"torchvision": "EomtImageProcessor", "pil": "EomtImageProcessorPil"}),