[staging CI] unslothai/unsloth#6978

tests/test_fp8_restore_dropped_scale.py

@@ -0,0 +1,328 @@
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Restoring dropped block-fp8 `weight_scale_inv` tensors on load (#6200).
+
+Some block-scale fp8 checkpoints leave a Linear (e.g. `mlp.gate_proj`) unconverted, so its raw
+quantized values land in a plain bf16 weight and its `weight_scale_inv` is dropped, producing a
+garbage un-scaled weight. `_restore_dropped_fp8_scales` dequantizes such orphaned weights in place
+using the scale from the checkpoint. Runs offline on CPU with synthetic checkpoints.
+"""
+
+import json
+import os
+import tempfile
+from types import SimpleNamespace
+
+import torch
+from torch import nn
+from safetensors.torch import save_file
+
+# Import unsloth first to set UNSLOTH_IS_PRESENT env var.
+import unsloth
+from unsloth.models.loader_utils import _restore_dropped_fp8_scales, _FP8_DTYPES
+
+
+_SHARD = "model-00001-of-00001.safetensors"
+_FP8 = _FP8_DTYPES[0] if _FP8_DTYPES else None
+
+
+def _write_checkpoint(
+    path,
+    tensors,
+    filename = _SHARD,
+    include_index = True,
+):
+    save_file(tensors, os.path.join(path, filename))
+    if include_index:
+        weight_map = {name: filename for name in tensors}
+        with open(os.path.join(path, "model.safetensors.index.json"), "w") as f:
+            json.dump({"weight_map": weight_map}, f)
+
+
+def _fp8_config(block = (2, 2)):
+    return SimpleNamespace(
+        quantization_config = {
+            "quant_method": "fp8",
+            "weight_block_size": list(block),
+        }
+    )
+
+
+def _fp8_anchor():
+    """A module carrying a real fp8 weight, so the model looks like a genuine fp8 load."""
+    m = nn.Linear(2, 2, bias = False)
+    m.weight = nn.Parameter(torch.randn(2, 2).to(_FP8), requires_grad = False)
+    return m
+
+
+def _bf16_linear(out_f, in_f, raw):
+    m = nn.Linear(in_f, out_f, bias = False).to(torch.bfloat16)
+    with torch.no_grad():
+        m.weight.copy_(raw)
+    return m
+
+
+def _expand(scale, block, shape):
+    bs0, bs1 = block
+    expanded = scale.repeat_interleave(bs0, dim = 0).repeat_interleave(bs1, dim = 1)
+    return expanded[: shape[0], : shape[1]]
+
+
+def test_restore_dequantizes_orphaned_scale():
+    """A plain bf16 weight whose scale was dropped is dequantized in place."""
+    if _FP8 is None:
+        return
+    torch.manual_seed(0)
+    raw = torch.randn(4, 4, dtype = torch.bfloat16)
+    scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
+
+    model = nn.Module()
+    model.config = _fp8_config((2, 2))
+    model.anchor = _fp8_anchor()
+    model.layer = _bf16_linear(4, 4, raw)
+
+    with tempfile.TemporaryDirectory() as d:
+        _write_checkpoint(
+            d,
+            {
+                "layer.weight": raw.to(torch.float32),
+                "layer.weight_scale_inv": scale,
+            },
+        )
+        restored, skipped = _restore_dropped_fp8_scales(model, d, local_files_only = True)
+
+    assert restored == 1
+    expected = (raw.to(torch.float32) * _expand(scale, (2, 2), (4, 4))).to(torch.bfloat16)
+    assert torch.equal(model.layer.weight.data, expected)
+
+
+def test_skips_already_fp8_weight():
+    """A correctly converted fp8 weight is skipped, never double-scaled."""
+    if _FP8 is None:
+        return
+    weight = torch.randn(4, 4).to(_FP8)
+    before = weight.clone()
+
+    model = nn.Module()
+    model.config = _fp8_config((2, 2))
+    model.layer = nn.Linear(4, 4, bias = False)
+    model.layer.weight = nn.Parameter(weight, requires_grad = False)
+
+    with tempfile.TemporaryDirectory() as d:
+        _write_checkpoint(d, {"layer.weight_scale_inv": torch.rand(2, 2)})
+        restored, skipped = _restore_dropped_fp8_scales(model, d, local_files_only = True)
+
+    assert restored == 0 and skipped == 1
+    assert torch.equal(model.layer.weight.data.float(), before.float())
+
+
+def test_noop_when_fully_dequantized():
+    """If the model has no fp8 weights at all (e.g. load_in_16bit dequantize), do not rescale."""
+    raw = torch.randn(4, 4, dtype = torch.bfloat16)
+    scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
+
+    model = nn.Module()
+    model.config = _fp8_config((2, 2))
+    model.layer = _bf16_linear(4, 4, raw)  # no fp8 anchor -> looks dequantized
+
+    with tempfile.TemporaryDirectory() as d:
+        _write_checkpoint(d, {"layer.weight_scale_inv": scale})
+        restored, skipped = _restore_dropped_fp8_scales(model, d, local_files_only = True)
+
+    assert (restored, skipped) == (0, 0)
+    assert torch.equal(model.layer.weight.data, raw)  # untouched
+
+
+def test_non_block_divisible_shape():
+    """Block scale is expanded then sliced to a non-divisible weight shape."""
+    if _FP8 is None:
+        return
+    raw = torch.randn(3, 4, dtype = torch.bfloat16)
+    scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
+
+    model = nn.Module()
+    model.config = _fp8_config((2, 2))
+    model.anchor = _fp8_anchor()
+    model.layer = _bf16_linear(3, 4, raw)  # weight shape [3, 4]
+
+    with tempfile.TemporaryDirectory() as d:
+        _write_checkpoint(d, {"layer.weight_scale_inv": scale})
+        restored, skipped = _restore_dropped_fp8_scales(model, d, local_files_only = True)
+
+    assert restored == 1
+    expected = (raw.to(torch.float32) * _expand(scale, (2, 2), (3, 4))).to(torch.bfloat16)
+    assert torch.equal(model.layer.weight.data, expected)
+
+
+def test_transposed_scale_layout():
+    """A scale stored in the transposed block grid is transposed before use."""
+    if _FP8 is None:
+        return
+    raw = torch.randn(4, 2, dtype = torch.bfloat16)  # weight [4, 2] -> grid (2, 1)
+    scale_correct = torch.rand(2, 1, dtype = torch.float32) + 0.1
+    scale_stored = scale_correct.t().contiguous()  # stored transposed as (1, 2)
+
+    model = nn.Module()
+    model.config = _fp8_config((2, 2))
+    model.anchor = _fp8_anchor()
+    model.layer = _bf16_linear(4, 2, raw)
+
+    with tempfile.TemporaryDirectory() as d:
+        _write_checkpoint(d, {"layer.weight_scale_inv": scale_stored})
+        restored, _ = _restore_dropped_fp8_scales(model, d, local_files_only = True)
+
+    assert restored == 1
+    expected = (raw.to(torch.float32) * _expand(scale_correct, (2, 2), (4, 2))).to(torch.bfloat16)
+    assert torch.equal(model.layer.weight.data, expected)
+
+
+def test_single_file_checkpoint_without_index():
+    """Unsharded model.safetensors (no index) is still scanned for dropped scales."""
+    if _FP8 is None:
+        return
+    raw = torch.randn(4, 4, dtype = torch.bfloat16)
+    scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
+
+    model = nn.Module()
+    model.config = _fp8_config((2, 2))
+    model.anchor = _fp8_anchor()
+    model.layer = _bf16_linear(4, 4, raw)
+
+    with tempfile.TemporaryDirectory() as d:
+        _write_checkpoint(
+            d, {"layer.weight_scale_inv": scale}, filename = "model.safetensors", include_index = False
+        )
+        restored, _ = _restore_dropped_fp8_scales(model, d, local_files_only = True)
+
+    assert restored == 1
+    expected = (raw.to(torch.float32) * _expand(scale, (2, 2), (4, 4))).to(torch.bfloat16)
+    assert torch.equal(model.layer.weight.data, expected)
+
+
+def test_scalar_block_size_config():
+    """A scalar weight_block_size (not a list) is handled without error."""
+    if _FP8 is None:
+        return
+    raw = torch.randn(4, 4, dtype = torch.bfloat16)
+    scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
+
+    model = nn.Module()
+    model.config = SimpleNamespace(
+        quantization_config = {"quant_method": "fp8", "weight_block_size": 2}
+    )
+    model.anchor = _fp8_anchor()
+    model.layer = _bf16_linear(4, 4, raw)
+
+    with tempfile.TemporaryDirectory() as d:
+        _write_checkpoint(d, {"layer.weight_scale_inv": scale})
+        restored, _ = _restore_dropped_fp8_scales(model, d, local_files_only = True)
+
+    assert restored == 1
+
+
+def test_text_only_prefix_mapping():
+    """Checkpoint keys with a language_model prefix match the stripped text-only module names."""
+    if _FP8 is None:
+        return
+    raw = torch.randn(2, 2, dtype = torch.bfloat16)
+    scale = torch.rand(1, 1, dtype = torch.float32) + 0.1
+
+    model = nn.Module()
+    model.config = _fp8_config((2, 2))
+    model.anchor = _fp8_anchor()
+    model.model = nn.Module()
+    model.model.gate_proj = _bf16_linear(2, 2, raw)  # module lacks the language_model prefix
+
+    with tempfile.TemporaryDirectory() as d:
+        # checkpoint key carries the language_model wrapper the text-only load stripped
+        _write_checkpoint(d, {"model.language_model.gate_proj.weight_scale_inv": scale})
+        restored, _ = _restore_dropped_fp8_scales(model, d, local_files_only = True)
+
+    assert restored == 1
+    expected = (raw.to(torch.float32) * _expand(scale, (2, 2), (2, 2))).to(torch.bfloat16)
+    assert torch.equal(model.model.gate_proj.weight.data, expected)
+
+
+def test_skips_variant_load():
+    """A variant load (variant="fp8") is skipped to avoid applying default-checkpoint scales."""
+    if _FP8 is None:
+        return
+    raw = torch.randn(4, 4, dtype = torch.bfloat16)
+    scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
+    model = nn.Module()
+    model.config = _fp8_config((2, 2))
+    model.anchor = _fp8_anchor()
+    model.layer = _bf16_linear(4, 4, raw)
+    with tempfile.TemporaryDirectory() as d:
+        _write_checkpoint(d, {"layer.weight_scale_inv": scale})
+        result = _restore_dropped_fp8_scales(model, d, local_files_only = True, variant = "fp8")
+    assert result == (0, 0)
+    assert torch.equal(model.layer.weight.data, raw)  # untouched
+
+
+def test_vlm_language_model_model_alias():
+    """A checkpoint key language_model.model.* matches a model.language_model.* module."""
+    if _FP8 is None:
+        return
+    raw = torch.randn(2, 2, dtype = torch.bfloat16)
+    scale = torch.rand(1, 1, dtype = torch.float32) + 0.1
+    model = nn.Module()
+    model.config = _fp8_config((2, 2))
+    model.anchor = _fp8_anchor()
+    model.model = nn.Module()
+    model.model.language_model = nn.Module()
+    model.model.language_model.gate_proj = _bf16_linear(
+        2, 2, raw
+    )  # -> model.language_model.gate_proj
+    with tempfile.TemporaryDirectory() as d:
+        _write_checkpoint(d, {"language_model.model.gate_proj.weight_scale_inv": scale})
+        restored, _ = _restore_dropped_fp8_scales(model, d, local_files_only = True)
+    assert restored == 1
+    expected = (raw.to(torch.float32) * _expand(scale, (2, 2), (2, 2))).to(torch.bfloat16)
+    assert torch.equal(model.model.language_model.gate_proj.weight.data, expected)
+
+
+def test_noop_without_scale_keys():
+    if _FP8 is None:
+        return
+    model = nn.Module()
+    model.config = _fp8_config((2, 2))
+    model.anchor = _fp8_anchor()
+    model.layer = _bf16_linear(4, 4, torch.randn(4, 4, dtype = torch.bfloat16))
+    with tempfile.TemporaryDirectory() as d:
+        _write_checkpoint(d, {"layer.weight": torch.randn(4, 4)})
+        assert _restore_dropped_fp8_scales(model, d, local_files_only = True) == (0, 0)
+
+
+def test_noop_without_index_or_single_file():
+    if _FP8 is None:
+        return
+    model = nn.Module()
+    model.config = _fp8_config((2, 2))
+    model.anchor = _fp8_anchor()
+    model.layer = _bf16_linear(4, 4, torch.randn(4, 4, dtype = torch.bfloat16))
+    with tempfile.TemporaryDirectory() as d:
+        assert _restore_dropped_fp8_scales(model, d, local_files_only = True) == (0, 0)
+
+
+def test_noop_when_not_block_fp8():
+    """A non-fp8 (or non-block) quantization config is ignored."""
+    scale = torch.rand(2, 2)
+    model = nn.Module()
+    model.config = SimpleNamespace(quantization_config = {"quant_method": "compressed-tensors"})
+    model.layer = nn.Linear(4, 4, bias = False)
+    with tempfile.TemporaryDirectory() as d:
+        _write_checkpoint(d, {"layer.weight_scale_inv": scale})
+        assert _restore_dropped_fp8_scales(model, d, local_files_only = True) == (0, 0)

unsloth/models/llama.py

@@ -28,7 +28,11 @@
     is_bfloat16_supported,
     get_quant_type,
 )
-from .loader_utils import _exclude_rope_inv_freq_from_ddp, _get_fp8_mode_and_check_settings
+from .loader_utils import (
+    _exclude_rope_inv_freq_from_ddp,
+    _get_fp8_mode_and_check_settings,
+    _restore_dropped_fp8_scales,
+)
 from ..utils.packing import (
     get_packed_info_from_kwargs,
     mask_packed_sequence_boundaries,
@@ -2678,6 +2682,17 @@ def from_pretrained(
                     offload_embedding = False,
                     fast_inference = fast_inference,
                 )
+                # Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200).
+                _restore_dropped_fp8_scales(
+                    model,
+                    model_name,
+                    local_files_only = kwargs.get("local_files_only", False),
+                    token = token,
+                    revision = revision,
+                    subfolder = kwargs.get("subfolder"),
+                    cache_dir = kwargs.get("cache_dir"),
+                    variant = kwargs.get("variant"),
+                )
             elif not fast_inference:
                 if user_config is not None:
                     # Transformers 5.x @strict model init rejects extra kwargs next
@@ -2716,6 +2731,17 @@ def from_pretrained(
                     offload_embedding = False,
                     fast_inference = False,
                 )
+                # Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200).
+                _restore_dropped_fp8_scales(
+                    model,
+                    model_name,
+                    local_files_only = kwargs.get("local_files_only", False),
+                    token = token,
+                    revision = revision,
+                    subfolder = kwargs.get("subfolder"),
+                    cache_dir = kwargs.get("cache_dir"),
+                    variant = kwargs.get("variant"),
+                )
                 model.fast_generate = make_fast_generate_wrapper(model.generate)
                 model.fast_generate_batches = None
             else: