From 8cbf9e9db027a02ceca0072a5cf50b61a0bcd144 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 29 Jun 2026 11:40:18 -0400 Subject: [PATCH 01/35] fix: preserve FP8 weight_scale_inv on load --- tests/test_fp8_scale_inv_load.py | 182 +++++++++++++++++++++++++++++++ unsloth/models/loader.py | 15 +++ unsloth/models/loader_utils.py | 156 ++++++++++++++++++++++++++ 3 files changed, 353 insertions(+) create mode 100644 tests/test_fp8_scale_inv_load.py diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py new file mode 100644 index 000000000000..c799e167d470 --- /dev/null +++ b/tests/test_fp8_scale_inv_load.py @@ -0,0 +1,182 @@ +import ast +import importlib.util +import os +from pathlib import Path +import sys +import tempfile +import types + +import torch +from safetensors.torch import save_file + + +REPO_ROOT = Path(__file__).resolve().parents[1] +LOADER_UTILS = REPO_ROOT / "unsloth" / "models" / "loader_utils.py" +LOADER = REPO_ROOT / "unsloth" / "models" / "loader.py" + + +def _install_stub_package(name, path): + module = types.ModuleType(name) + module.__path__ = [str(path)] + sys.modules[name] = module + return module + + +def _install_loader_utils_stubs(): + _install_stub_package("unsloth", REPO_ROOT / "unsloth") + _install_stub_package("unsloth.models", REPO_ROOT / "unsloth" / "models") + _install_stub_package("unsloth_zoo", REPO_ROOT) + + device_type = types.ModuleType("unsloth.device_type") + device_type.DEVICE_TYPE_TORCH = "cuda" + sys.modules["unsloth.device_type"] = device_type + + mapper = types.ModuleType("unsloth.models.mapper") + mapper.INT_TO_FLOAT_MAPPER = {} + mapper.FLOAT_TO_INT_MAPPER = {} + mapper.MAP_TO_UNSLOTH_16bit = {} + mapper.FLOAT_TO_FP8_BLOCK_MAPPER = {} + mapper.FLOAT_TO_FP8_ROW_MAPPER = {} + sys.modules["unsloth.models.mapper"] = mapper + + model_utils = types.ModuleType("unsloth.models._utils") + model_utils.TorchAOConfig = type("TorchAOConfig", (), {}) + sys.modules["unsloth.models._utils"] = model_utils + + zoo_utils = types.ModuleType("unsloth_zoo.utils") + zoo_utils.Version = lambda value: value + zoo_utils.get_quant_type = lambda config: None + sys.modules["unsloth_zoo.utils"] = zoo_utils + + +def _load_loader_utils(): + _install_loader_utils_stubs() + sys.modules.pop("unsloth.models.loader_utils", None) + spec = importlib.util.spec_from_file_location("unsloth.models.loader_utils", LOADER_UTILS) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _make_checkpoint(path, tensors): + full_path = os.path.join(path, "model-00001-of-00001.safetensors") + save_file(tensors, full_path) + return path + + +class _Fp8Owner(torch.nn.Module): + def __init__(self, weight, quant_method = "fp8", weight_scale = None): + super().__init__() + self.weight = torch.nn.Parameter(weight) + self.quant_method = quant_method + if weight_scale is not None: + self.weight_scale = weight_scale + + +def test_restores_missing_fp8_weight_scale_inv(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.25], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([1.25], dtype = torch.float16)) + assert not hasattr(model.fp8, "weight_scale") + + +def test_does_not_attach_scale_inv_to_non_fp8_module(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.linear = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16), quant_method = "int8") + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"linear.weight_scale_inv": torch.tensor([0.5], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 0 + assert skipped == 1 + assert not hasattr(model.linear, "weight_scale_inv") + + +def test_preserves_existing_scale_state(): + loader_utils = _load_loader_utils() + weight_scale = torch.tensor([3.0], dtype = torch.float16) + model = torch.nn.Module() + model.fp8 = _Fp8Owner( + weight = torch.randn(4, 4, dtype = torch.float16), + weight_scale = weight_scale, + ) + existing_inv = torch.tensor([5.0], dtype = torch.float16) + model.fp8.weight_scale_inv = existing_inv + weight_scale_id = id(model.fp8.weight_scale) + weight_scale_inv_id = id(model.fp8.weight_scale_inv) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.0], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 0 + assert skipped == 1 + assert id(model.fp8.weight_scale_inv) == weight_scale_inv_id + assert id(model.fp8.weight_scale) == weight_scale_id + assert torch.equal(model.fp8.weight_scale, weight_scale) + assert torch.equal(model.fp8.weight_scale_inv, existing_inv) + + +def test_loader_calls_restore_only_for_fp8_loads(): + tree = ast.parse(LOADER.read_text()) + hit_count = 0 + + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + test = node.test + if not ( + isinstance(test, ast.Compare) + and isinstance(test.left, ast.Name) + and test.left.id == "load_in_fp8" + and len(test.ops) == 1 + and isinstance(test.ops[0], ast.NotEq) + and len(test.comparators) == 1 + and isinstance(test.comparators[0], ast.Constant) + and test.comparators[0].value is False + ): + continue + + call_names = set() + for child in ast.walk(node): + if isinstance(child, ast.Call) and isinstance(child.func, ast.Name): + call_names.add(child.func.id) + if { + "_restore_missing_fp8_weight_scale_inv", + "_tag_model_with_fp8_torchao_config", + }.issubset(call_names): + hit_count += 1 + + assert hit_count == 2 diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 562afdd645d2..8f5b31754867 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -36,6 +36,7 @@ _exclude_rope_inv_freq_from_ddp, _get_fp8_mode_and_check_settings, _offline_quantize_to_fp8, + _restore_missing_fp8_weight_scale_inv, _tag_model_with_fp8_torchao_config, get_model_name, prepare_device_map, @@ -860,6 +861,13 @@ def from_pretrained( model.config.update({"quantization_config": quantization_config}) if load_in_fp8 != False: + _restored_count, _skipped_count = _restore_missing_fp8_weight_scale_inv( + model, + model_name, + token = token, + revision = revision, + local_files_only = local_files_only, + ) _tag_model_with_fp8_torchao_config(model, fp8_mode) if is_peft: @@ -1733,6 +1741,13 @@ def _dispatch_diffusion(): model.config.update({"quantization_config": quantization_config}) if load_in_fp8 != False: + _restored_count, _skipped_count = _restore_missing_fp8_weight_scale_inv( + model, + model_name, + token = token, + revision = revision, + local_files_only = local_files_only, + ) _tag_model_with_fp8_torchao_config(model, fp8_mode) if is_peft: diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index d6d6ce877e05..13c0dafa36c1 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -362,6 +362,162 @@ def _tag_model_with_fp8_torchao_config(model: torch.nn.Module, fp8_mode: str): pass +def _has_float8_dtype(dtype): + dtype_name = str(dtype) + return dtype_name.startswith("torch.float8_") + + +def _is_real_fp8_owner(module): + if not isinstance(module, torch.nn.Module): + return False + weight = getattr(module, "weight", None) + if not isinstance(weight, torch.Tensor): + return False + if getattr(module, "quant_method", None) in ["fp8", "fbgemm_fp8"]: + return True + if getattr(weight, "quant_method", None) in ["fp8", "fbgemm_fp8"]: + return True + + config = getattr(module, "quantization_config", None) + if isinstance(config, dict) and config.get("quant_method") in ["fp8", "fbgemm_fp8"]: + return True + if getattr(config, "quant_method", None) in ["fp8", "fbgemm_fp8"]: + return True + + if _has_float8_dtype(getattr(weight, "dtype", None)): + return True + + try: + from ..kernels.fp8 import FbgemmFp8Linear, FP8Linear + except Exception: + FbgemmFp8Linear, FP8Linear = None, None + if FbgemmFp8Linear is not None and isinstance(module, FbgemmFp8Linear): + return True + if FP8Linear is not None and isinstance(module, FP8Linear): + return True + + return False + + +def _resolve_weight_scale_inv_candidates(module_name, named_modules): + if module_name in named_modules: + return module_name + suffix = f".{module_name}" + suffix_matches = [name for name in named_modules if name.endswith(suffix)] + if len(suffix_matches) == 1: + return suffix_matches[0] + return None + + +def _find_fp8_scale_inv_tensors(model_dir, model_name, revision = None, token = None, local_files_only = False): + if not os.path.exists(model_dir): + try: + from huggingface_hub import snapshot_download + except Exception: + return [] + + try: + model_dir = snapshot_download( + model_name, + revision = revision, + token = token, + local_files_only = local_files_only, + ) + except Exception: + return [] + + if not os.path.isdir(model_dir): + return [] + + try: + from safetensors.torch import safe_open + except Exception: + return [] + + module_scales = {} + suffix = ".weight_scale_inv" + for filename in os.listdir(model_dir): + if not filename.endswith(".safetensors"): + continue + full_path = os.path.join(model_dir, filename) + try: + with safe_open(full_path, framework = "pt", device = "cpu") as f: + for key in f.keys(): + if not key.endswith(suffix): + continue + module_name = key[: -len(suffix)] + module_scales[module_name] = f.get_tensor(key) + except Exception: + continue + return list(module_scales.items()) + + +def _restore_missing_fp8_weight_scale_inv( + model: torch.nn.Module, + model_name: str, + token = None, + revision = None, + local_files_only = False, +): + """Find checkpointed `.weight_scale_inv` tensors and restore missing runtime tensors on FP8 modules. + + Returns a tuple of `(restored_count, skipped_count)`. + """ + if not hasattr(model, "named_modules"): + return 0, 0 + + name_to_module = dict(model.named_modules()) + scaled_tensors = _find_fp8_scale_inv_tensors( + model_dir = model_name, + model_name = model_name, + revision = revision, + token = token, + local_files_only = local_files_only, + ) + if len(scaled_tensors) == 0: + return 0, 0 + + restored = 0 + skipped = 0 + for module_name, scale_tensor in scaled_tensors: + target_name = _resolve_weight_scale_inv_candidates(module_name, name_to_module) + if target_name is None: + skipped += 1 + continue + + module = name_to_module[target_name] + if not _is_real_fp8_owner(module): + skipped += 1 + continue + + if hasattr(module, "weight_scale_inv"): + skipped += 1 + continue + + weight = getattr(module, "weight", None) + if not isinstance(weight, torch.Tensor): + skipped += 1 + continue + + restored_scale = scale_tensor.to(device = weight.device) + reference = getattr(module, "weight_scale", None) + if isinstance(reference, torch.Tensor): + try: + restored_scale = restored_scale.to(dtype = reference.dtype) + except Exception: + pass + elif not _has_float8_dtype(getattr(weight, "dtype", None)): + try: + restored_scale = restored_scale.to(dtype = weight.dtype) + except Exception: + pass + + setattr(module, "weight_scale_inv", restored_scale) + restored += 1 + + return restored, skipped + + def check_and_disable_bitsandbytes_loading( model_config, load_in_4bit = True, From 0ca7ca40e1feddad0f0da86bdc2a5a1e78421960 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 29 Jun 2026 11:53:36 -0400 Subject: [PATCH 02/35] fix: register restored FP8 inverse scales --- tests/test_fp8_scale_inv_load.py | 9 ++++++++- unsloth/models/loader_utils.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index c799e167d470..71a99bca6c99 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -93,6 +93,11 @@ def test_restores_missing_fp8_weight_scale_inv(): assert restored == 1 assert skipped == 0 assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([1.25], dtype = torch.float16)) + assert "fp8.weight_scale_inv" in model.state_dict() + assert torch.equal( + model.state_dict()["fp8.weight_scale_inv"], + torch.tensor([1.25], dtype = torch.float16), + ) assert not hasattr(model.fp8, "weight_scale") @@ -126,7 +131,7 @@ def test_preserves_existing_scale_state(): weight_scale = weight_scale, ) existing_inv = torch.tensor([5.0], dtype = torch.float16) - model.fp8.weight_scale_inv = existing_inv + model.fp8.register_buffer("weight_scale_inv", existing_inv) weight_scale_id = id(model.fp8.weight_scale) weight_scale_inv_id = id(model.fp8.weight_scale_inv) @@ -147,6 +152,8 @@ def test_preserves_existing_scale_state(): assert id(model.fp8.weight_scale) == weight_scale_id assert torch.equal(model.fp8.weight_scale, weight_scale) assert torch.equal(model.fp8.weight_scale_inv, existing_inv) + assert "fp8.weight_scale_inv" in model.state_dict() + assert torch.equal(model.state_dict()["fp8.weight_scale_inv"], existing_inv) def test_loader_calls_restore_only_for_fp8_loads(): diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 13c0dafa36c1..14d999483471 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -512,7 +512,7 @@ def _restore_missing_fp8_weight_scale_inv( except Exception: pass - setattr(module, "weight_scale_inv", restored_scale) + module.register_buffer("weight_scale_inv", restored_scale) restored += 1 return restored, skipped From a2640a03e35f4d48902cda5428519aa0c58b8e04 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 29 Jun 2026 12:11:50 -0400 Subject: [PATCH 03/35] fix: keep optional FP8 kernel imports non-fatal --- tests/test_fp8_scale_inv_load.py | 36 ++++++++++++++++++++++++++++++++ unsloth/models/loader_utils.py | 4 ++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 71a99bca6c99..42c3267a6156 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -6,6 +6,7 @@ import tempfile import types +import pytest import torch from safetensors.torch import save_file @@ -187,3 +188,38 @@ def test_loader_calls_restore_only_for_fp8_loads(): hit_count += 1 assert hit_count == 2 + + +def test_fp8_probe_skips_missing_optional_kernel_module(): + loader_utils = _load_loader_utils() + module = torch.nn.Linear(4, 4, bias = False) + + assert loader_utils._is_real_fp8_owner(module) is False + + +def test_fp8_probe_uses_absolute_optional_import_and_narrow_missing_module_guard(): + tree = ast.parse(LOADER_UTILS.read_text()) + + for node in ast.walk(tree): + if not isinstance(node, ast.FunctionDef) or node.name != "_is_real_fp8_owner": + continue + for child in ast.walk(node): + if not isinstance(child, ast.Try): + continue + + imports = [ + stmt for stmt in child.body + if isinstance(stmt, ast.ImportFrom) + ] + handlers = child.handlers + if not imports or not handlers: + continue + + import_stmt = imports[0] + handler = handlers[0] + assert import_stmt.module == "unsloth.kernels.fp8" + assert isinstance(handler.type, ast.Name) + assert handler.type.id == "ModuleNotFoundError" + return + + pytest.fail("_is_real_fp8_owner no longer protects the optional fp8 kernel import.") diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 14d999483471..d726fa3e1948 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -388,8 +388,8 @@ def _is_real_fp8_owner(module): return True try: - from ..kernels.fp8 import FbgemmFp8Linear, FP8Linear - except Exception: + from unsloth.kernels.fp8 import FbgemmFp8Linear, FP8Linear + except ModuleNotFoundError: FbgemmFp8Linear, FP8Linear = None, None if FbgemmFp8Linear is not None and isinstance(module, FbgemmFp8Linear): return True From a8e44745caf568749f2d1abc11fe211e1fb725d4 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 29 Jun 2026 12:16:50 -0400 Subject: [PATCH 04/35] fix: restore mapped FP8 inverse scales --- tests/test_fp8_scale_inv_load.py | 73 +++++++++++++++++++++++++------- unsloth/models/loader.py | 8 +++- unsloth/models/loader_utils.py | 27 ++++++------ 3 files changed, 78 insertions(+), 30 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 42c3267a6156..76cf91766026 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -23,6 +23,27 @@ def _install_stub_package(name, path): return module +@pytest.fixture(autouse = True) +def _restore_loader_utils_modules(): + keep = { + "unsloth", + "unsloth.models", + "unsloth.models.loader_utils", + "unsloth.device_type", + "unsloth.models.mapper", + "unsloth.models._utils", + "unsloth_zoo", + "unsloth_zoo.utils", + } + snapshot = {name: sys.modules.get(name) for name in keep} + yield + for name, value in snapshot.items(): + if value is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = value + + def _install_loader_utils_stubs(): _install_stub_package("unsloth", REPO_ROOT / "unsloth") _install_stub_package("unsloth.models", REPO_ROOT / "unsloth" / "models") @@ -147,14 +168,45 @@ def test_preserves_existing_scale_state(): local_files_only = True, ) - assert restored == 0 - assert skipped == 1 - assert id(model.fp8.weight_scale_inv) == weight_scale_inv_id + assert restored == 1 + assert skipped == 0 assert id(model.fp8.weight_scale) == weight_scale_id assert torch.equal(model.fp8.weight_scale, weight_scale) - assert torch.equal(model.fp8.weight_scale_inv, existing_inv) + assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([1.0], dtype = torch.float16)) assert "fp8.weight_scale_inv" in model.state_dict() - assert torch.equal(model.state_dict()["fp8.weight_scale_inv"], existing_inv) + assert torch.equal( + model.state_dict()["fp8.weight_scale_inv"], + torch.tensor([1.0], dtype = torch.float16), + ) + + +class _PackedFp8Owner(torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.ones(4, 4, dtype = torch.int8) + self.quant_method = "fp8" + + +def test_preserves_checkpoint_scale_dtype_for_packed_fp8_weights(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _PackedFp8Owner() + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.25], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert model.fp8.weight_scale_inv.dtype == torch.float32 + assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([1.25], dtype = torch.float32)) def test_loader_calls_restore_only_for_fp8_loads(): @@ -165,16 +217,7 @@ def test_loader_calls_restore_only_for_fp8_loads(): if not isinstance(node, ast.If): continue test = node.test - if not ( - isinstance(test, ast.Compare) - and isinstance(test.left, ast.Name) - and test.left.id == "load_in_fp8" - and len(test.ops) == 1 - and isinstance(test.ops[0], ast.NotEq) - and len(test.comparators) == 1 - and isinstance(test.comparators[0], ast.Constant) - and test.comparators[0].value is False - ): + if not isinstance(test, ast.Name) or test.id != "restore_fp8_scales": continue call_names = set() diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 8f5b31754867..3e12da5128e2 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -438,6 +438,7 @@ def from_pretrained( # Find FP8, BnB 4bit, other mapped names old_model_name = model_name fp8_mode = None + restore_fp8_scales = load_in_fp8 != False if not use_exact_model_name: new_model_name = get_model_name( model_name, @@ -475,6 +476,7 @@ def from_pretrained( load_in_4bit = False load_in_8bit = False load_in_fp8 = False + restore_fp8_scales = False load_in_16bit = True if USE_MODELSCOPE and not os.path.exists(model_name): @@ -860,7 +862,7 @@ def from_pretrained( elif isinstance(quantization_config, dict): model.config.update({"quantization_config": quantization_config}) - if load_in_fp8 != False: + if restore_fp8_scales: _restored_count, _skipped_count = _restore_missing_fp8_weight_scale_inv( model, model_name, @@ -1096,6 +1098,7 @@ def from_pretrained( # Find FP8, BnB 4bit, other mapped names old_model_name = model_name fp8_mode = None + restore_fp8_scales = load_in_fp8 != False if not use_exact_model_name: new_model_name = get_model_name( model_name, load_in_4bit = load_in_4bit, load_in_fp8 = load_in_fp8 @@ -1129,6 +1132,7 @@ def from_pretrained( load_in_4bit = False load_in_8bit = False load_in_fp8 = False + restore_fp8_scales = False load_in_16bit = True # Check modelscope @@ -1740,7 +1744,7 @@ def _dispatch_diffusion(): elif isinstance(quantization_config, dict): model.config.update({"quantization_config": quantization_config}) - if load_in_fp8 != False: + if restore_fp8_scales: _restored_count, _skipped_count = _restore_missing_fp8_weight_scale_inv( model, model_name, diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index d726fa3e1948..8d4e9dc5f1f5 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -14,6 +14,7 @@ from ..device_type import DEVICE_TYPE_TORCH import importlib +import logging import os import torch import re @@ -36,6 +37,8 @@ from unsloth_zoo.utils import Version, get_quant_type import gc +logger = logging.getLogger(__name__) + transformers_version = Version(transformers_version) SUPPORTS_FOURBIT = transformers_version >= Version("4.37") @@ -389,7 +392,8 @@ def _is_real_fp8_owner(module): try: from unsloth.kernels.fp8 import FbgemmFp8Linear, FP8Linear - except ModuleNotFoundError: + except ModuleNotFoundError as exc: + logger.debug("Optional fp8 kernels not available: %s", exc) FbgemmFp8Linear, FP8Linear = None, None if FbgemmFp8Linear is not None and isinstance(module, FbgemmFp8Linear): return True @@ -490,29 +494,26 @@ def _restore_missing_fp8_weight_scale_inv( skipped += 1 continue - if hasattr(module, "weight_scale_inv"): - skipped += 1 - continue - weight = getattr(module, "weight", None) if not isinstance(weight, torch.Tensor): skipped += 1 continue restored_scale = scale_tensor.to(device = weight.device) - reference = getattr(module, "weight_scale", None) + reference = getattr(module, "weight_scale_inv", None) + if not isinstance(reference, torch.Tensor): + reference = getattr(module, "weight_scale", None) if isinstance(reference, torch.Tensor): try: restored_scale = restored_scale.to(dtype = reference.dtype) except Exception: pass - elif not _has_float8_dtype(getattr(weight, "dtype", None)): - try: - restored_scale = restored_scale.to(dtype = weight.dtype) - except Exception: - pass - - module.register_buffer("weight_scale_inv", restored_scale) + if "weight_scale_inv" in module._buffers: + module._buffers["weight_scale_inv"] = restored_scale + else: + if hasattr(module, "weight_scale_inv"): + delattr(module, "weight_scale_inv") + module.register_buffer("weight_scale_inv", restored_scale) restored += 1 return restored, skipped From cf0e821fa4d3ae63ec18fff07ff189313c0a326d Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 29 Jun 2026 12:38:41 -0400 Subject: [PATCH 05/35] fix: restore exact FP8 checkpoint scales --- tests/test_fp8_scale_inv_load.py | 79 +++++++++++++++++++++++++++++++- unsloth/models/loader.py | 9 ++-- unsloth/models/loader_utils.py | 40 ++++++++++++---- 3 files changed, 114 insertions(+), 14 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 76cf91766026..eb140c01fbbe 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -87,6 +87,12 @@ def _make_checkpoint(path, tensors): return path +def _make_bin_checkpoint(path, tensors): + full_path = os.path.join(path, "pytorch_model-00001-of-00001.bin") + torch.save(tensors, full_path) + return path + + class _Fp8Owner(torch.nn.Module): def __init__(self, weight, quant_method = "fp8", weight_scale = None): super().__init__() @@ -209,7 +215,28 @@ def test_preserves_checkpoint_scale_dtype_for_packed_fp8_weights(): assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([1.25], dtype = torch.float32)) -def test_loader_calls_restore_only_for_fp8_loads(): +def test_restores_missing_fp8_weight_scale_inv_from_bin_checkpoint(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _PackedFp8Owner() + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_bin_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([2.5], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([2.5], dtype = torch.float32)) + + +def test_loader_detects_direct_or_requested_fp8_restore_paths(): tree = ast.parse(LOADER.read_text()) hit_count = 0 @@ -217,7 +244,16 @@ def test_loader_calls_restore_only_for_fp8_loads(): if not isinstance(node, ast.If): continue test = node.test - if not isinstance(test, ast.Name) or test.id != "restore_fp8_scales": + if not ( + isinstance(test, ast.BoolOp) + and isinstance(test.op, ast.Or) + and len(test.values) == 2 + and isinstance(test.values[0], ast.Name) + and test.values[0].id == "restore_fp8_scales" + and isinstance(test.values[1], ast.Call) + and isinstance(test.values[1].func, ast.Name) + and test.values[1].func.id == "_model_has_real_fp8_modules" + ): continue call_names = set() @@ -233,6 +269,37 @@ def test_loader_calls_restore_only_for_fp8_loads(): assert hit_count == 2 +def test_loader_restores_fp8_scales_with_base_revision_for_peft(): + tree = ast.parse(LOADER.read_text()) + hit_count = 0 + + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if not isinstance(node.func, ast.Name) or node.func.id != "_restore_missing_fp8_weight_scale_inv": + continue + + revision_kw = next((kw for kw in node.keywords if kw.arg == "revision"), None) + if revision_kw is None: + continue + value = revision_kw.value + if not ( + isinstance(value, ast.IfExp) + and isinstance(value.test, ast.UnaryOp) + and isinstance(value.test.op, ast.Not) + and isinstance(value.test.operand, ast.Name) + and value.test.operand.id == "is_peft" + and isinstance(value.body, ast.Name) + and value.body.id == "revision" + and isinstance(value.orelse, ast.Constant) + and value.orelse.value is None + ): + continue + hit_count += 1 + + assert hit_count == 2 + + def test_fp8_probe_skips_missing_optional_kernel_module(): loader_utils = _load_loader_utils() module = torch.nn.Linear(4, 4, bias = False) @@ -240,6 +307,14 @@ def test_fp8_probe_skips_missing_optional_kernel_module(): assert loader_utils._is_real_fp8_owner(module) is False +def test_model_has_real_fp8_modules_detects_nested_fp8_layers(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _PackedFp8Owner() + + assert loader_utils._model_has_real_fp8_modules(model) is True + + def test_fp8_probe_uses_absolute_optional_import_and_narrow_missing_module_guard(): tree = ast.parse(LOADER_UTILS.read_text()) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 3e12da5128e2..cc8e24ee6045 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -35,6 +35,7 @@ from .loader_utils import ( _exclude_rope_inv_freq_from_ddp, _get_fp8_mode_and_check_settings, + _model_has_real_fp8_modules, _offline_quantize_to_fp8, _restore_missing_fp8_weight_scale_inv, _tag_model_with_fp8_torchao_config, @@ -862,12 +863,12 @@ def from_pretrained( elif isinstance(quantization_config, dict): model.config.update({"quantization_config": quantization_config}) - if restore_fp8_scales: + if restore_fp8_scales or _model_has_real_fp8_modules(model): _restored_count, _skipped_count = _restore_missing_fp8_weight_scale_inv( model, model_name, token = token, - revision = revision, + revision = revision if not is_peft else None, local_files_only = local_files_only, ) _tag_model_with_fp8_torchao_config(model, fp8_mode) @@ -1744,12 +1745,12 @@ def _dispatch_diffusion(): elif isinstance(quantization_config, dict): model.config.update({"quantization_config": quantization_config}) - if restore_fp8_scales: + if restore_fp8_scales or _model_has_real_fp8_modules(model): _restored_count, _skipped_count = _restore_missing_fp8_weight_scale_inv( model, model_name, token = token, - revision = revision, + revision = revision if not is_peft else None, local_files_only = local_files_only, ) _tag_model_with_fp8_torchao_config(model, fp8_mode) diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 8d4e9dc5f1f5..4c0761e57d0f 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -403,6 +403,15 @@ def _is_real_fp8_owner(module): return False +def _model_has_real_fp8_modules(model): + if not hasattr(model, "modules"): + return False + for module in model.modules(): + if _is_real_fp8_owner(module): + return True + return False + + def _resolve_weight_scale_inv_candidates(module_name, named_modules): if module_name in named_modules: return module_name @@ -441,16 +450,31 @@ def _find_fp8_scale_inv_tensors(model_dir, model_name, revision = None, token = module_scales = {} suffix = ".weight_scale_inv" for filename in os.listdir(model_dir): - if not filename.endswith(".safetensors"): - continue full_path = os.path.join(model_dir, filename) + if filename.endswith(".safetensors"): + try: + with safe_open(full_path, framework = "pt", device = "cpu") as f: + for key in f.keys(): + if not key.endswith(suffix): + continue + module_name = key[: -len(suffix)] + module_scales[module_name] = f.get_tensor(key) + except Exception: + continue + continue + if not filename.endswith(".bin"): + continue try: - with safe_open(full_path, framework = "pt", device = "cpu") as f: - for key in f.keys(): - if not key.endswith(suffix): - continue - module_name = key[: -len(suffix)] - module_scales[module_name] = f.get_tensor(key) + state_dict = torch.load(full_path, map_location = "cpu") + if isinstance(state_dict, dict) and isinstance(state_dict.get("state_dict"), dict): + state_dict = state_dict["state_dict"] + if not isinstance(state_dict, dict): + continue + for key, value in state_dict.items(): + if not key.endswith(suffix) or not isinstance(value, torch.Tensor): + continue + module_name = key[: -len(suffix)] + module_scales[module_name] = value except Exception: continue return list(module_scales.items()) From 41c99189ef67b5a1e8459de376ea93f086599f9c Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Tue, 7 Jul 2026 02:31:44 -0400 Subject: [PATCH 06/35] fix: constrain FP8 scale restore to selected safetensors --- tests/test_fp8_scale_inv_load.py | 242 ++++++++++++++++++++++++++++++- unsloth/models/loader.py | 24 ++- unsloth/models/loader_utils.py | 174 +++++++++++++++++----- 3 files changed, 394 insertions(+), 46 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index eb140c01fbbe..96788de8b916 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1,5 +1,6 @@ import ast import importlib.util +import json import os from pathlib import Path import sys @@ -7,6 +8,7 @@ import types import pytest +import safetensors.torch import torch from safetensors.torch import save_file @@ -81,8 +83,8 @@ def _load_loader_utils(): return module -def _make_checkpoint(path, tensors): - full_path = os.path.join(path, "model-00001-of-00001.safetensors") +def _make_checkpoint(path, tensors, filename = "model.safetensors"): + full_path = os.path.join(path, filename) save_file(tensors, full_path) return path @@ -215,7 +217,7 @@ def test_preserves_checkpoint_scale_dtype_for_packed_fp8_weights(): assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([1.25], dtype = torch.float32)) -def test_restores_missing_fp8_weight_scale_inv_from_bin_checkpoint(): +def test_skips_bin_checkpoint_without_torch_load(monkeypatch): loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _PackedFp8Owner() @@ -225,15 +227,190 @@ def test_restores_missing_fp8_weight_scale_inv_from_bin_checkpoint(): checkpoint_dir, {"fp8.weight_scale_inv": torch.tensor([2.5], dtype = torch.float32)}, ) + monkeypatch.setattr(loader_utils.torch, "load", lambda *args, **kwargs: pytest.fail("torch.load should not run")) restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( model, model_name = checkpoint_dir, local_files_only = True, ) + assert restored == 0 + assert skipped == 0 + assert not hasattr(model.fp8, "weight_scale_inv") + + +def test_variant_limits_scale_restore_to_selected_safetensors(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _PackedFp8Owner() + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.0], dtype = torch.float32)}, + filename = "model.safetensors", + ) + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([3.0], dtype = torch.float32)}, + filename = "model.fp8.safetensors", + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + variant = "fp8", + ) + assert restored == 1 assert skipped == 0 - assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([2.5], dtype = torch.float32)) + assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([3.0], dtype = torch.float32)) + + +def test_subfolder_limits_scale_restore_to_selected_safetensors(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _PackedFp8Owner() + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.0], dtype = torch.float32)}, + ) + subfolder = Path(checkpoint_dir) / "nested" + subfolder.mkdir() + _make_checkpoint( + subfolder, + {"fp8.weight_scale_inv": torch.tensor([4.0], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + subfolder = "nested", + ) + + assert restored == 1 + assert skipped == 0 + assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([4.0], dtype = torch.float32)) + + +def test_safetensors_index_opens_only_scale_shards(monkeypatch): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _PackedFp8Owner() + opened = [] + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([5.0], dtype = torch.float32)}, + filename = "scale.safetensors", + ) + _make_checkpoint( + checkpoint_dir, + {"other.weight": torch.tensor([9.0], dtype = torch.float32)}, + filename = "other.safetensors", + ) + index = { + "weight_map": { + "fp8.weight_scale_inv": "scale.safetensors", + "other.weight": "other.safetensors", + } + } + (Path(checkpoint_dir) / "model.safetensors.index.json").write_text(json.dumps(index)) + def recording_safe_open(path, *args, **kwargs): + opened.append(Path(path).name) + return original_safe_open(path, *args, **kwargs) + + original_safe_open = safetensors.torch.safe_open + monkeypatch.setattr(safetensors.torch, "safe_open", recording_safe_open) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert opened == ["scale.safetensors"] + + +def test_skips_meta_device_scale_restore(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.empty(4, 4, device = "meta"), weight_scale = torch.empty(1, device = "meta")) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.0], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 0 + assert skipped == 1 + assert not hasattr(model.fp8, "weight_scale_inv") + + +def test_skips_shape_incompatible_local_scale(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner( + weight = torch.randn(4, 4, dtype = torch.float16), + weight_scale = torch.ones(2, dtype = torch.float16), + ) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.ones(4, dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 0 + assert skipped == 1 + assert not hasattr(model.fp8, "weight_scale_inv") + + +class _Fp8Expert(torch.nn.Module): + def __init__(self): + super().__init__() + self.gate_up_proj_scale_inv = torch.ones(2, dtype = torch.float16) + self.down_proj_scale_inv = torch.ones(2, dtype = torch.float16) + + +def test_restores_fp8_expert_scale_tensors(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.expert = _Fp8Expert() + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + { + "expert.gate_up_proj_scale_inv": torch.full((2,), 2.0, dtype = torch.float32), + "expert.down_proj_scale_inv": torch.full((2,), 3.0, dtype = torch.float32), + }, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 2 + assert skipped == 0 + assert torch.equal(model.expert.gate_up_proj_scale_inv, torch.full((2,), 2.0, dtype = torch.float16)) + assert torch.equal(model.expert.down_proj_scale_inv, torch.full((2,), 3.0, dtype = torch.float16)) def test_loader_detects_direct_or_requested_fp8_restore_paths(): @@ -300,6 +477,63 @@ def test_loader_restores_fp8_scales_with_base_revision_for_peft(): assert hit_count == 2 +def test_loader_passes_selected_artifact_knobs_to_fp8_restore(): + tree = ast.parse(LOADER.read_text()) + hit_count = 0 + + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if not isinstance(node.func, ast.Name) or node.func.id != "_restore_missing_fp8_weight_scale_inv": + continue + keyword_names = {kw.arg for kw in node.keywords} + if {"subfolder", "variant", "use_safetensors"}.issubset(keyword_names): + hit_count += 1 + + assert hit_count == 2 + + +def test_fastmodel_peft_base_mapping_forwards_load_in_fp8(): + tree = ast.parse(LOADER.read_text()) + + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + if not isinstance(node.test, ast.Name) or node.test.id != "is_peft": + continue + for child in ast.walk(node): + if not isinstance(child, ast.Assign): + continue + if not any(isinstance(target, ast.Name) and target.id == "model_name" for target in child.targets): + continue + value = child.value + if not isinstance(value, ast.Call): + continue + if not isinstance(value.func, ast.Name) or value.func.id != "get_model_name": + continue + keyword_names = {kw.arg for kw in value.keywords} + if {"load_in_4bit", "load_in_fp8"}.issubset(keyword_names): + return + + pytest.fail("FastModel PEFT base mapping must forward load_in_fp8 to get_model_name.") + + +def test_offline_fp8_cache_uses_safe_serialization(): + tree = ast.parse(LOADER_UTILS.read_text()) + + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if not isinstance(node.func, ast.Attribute) or node.func.attr != "save_pretrained": + continue + for kw in node.keywords: + if kw.arg == "safe_serialization" and isinstance(kw.value, ast.Constant): + assert kw.value.value is True + return + + pytest.fail("_offline_quantize_to_fp8 must save safetensors caches.") + + def test_fp8_probe_skips_missing_optional_kernel_module(): loader_utils = _load_loader_utils() module = torch.nn.Linear(4, 4, bias = False) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index cc8e24ee6045..3302383d1640 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -439,7 +439,7 @@ def from_pretrained( # Find FP8, BnB 4bit, other mapped names old_model_name = model_name fp8_mode = None - restore_fp8_scales = load_in_fp8 != False + restore_fp8_scales = False if not use_exact_model_name: new_model_name = get_model_name( model_name, @@ -458,12 +458,14 @@ def from_pretrained( load_in_16bit, ) model_name = _offline_quantize_to_fp8(model_name, fp8_mode, text_only = text_only) + restore_fp8_scales = True else: assert new_model_name is not None model_name = new_model_name # If mapper resolved to a pre-quantized FP8 model, disable # on-the-fly quantization to avoid double quantization if load_in_fp8 != False and new_model_name != old_model_name: + restore_fp8_scales = True load_in_fp8 = False # Check if pre-quantized models are allowed @@ -870,6 +872,11 @@ def from_pretrained( token = token, revision = revision if not is_peft else None, local_files_only = local_files_only, + subfolder = kwargs.get("subfolder"), + variant = kwargs.get("variant"), + use_safetensors = kwargs.get("use_safetensors"), + cache_dir = kwargs.get("cache_dir"), + force_download = kwargs.get("force_download", False), ) _tag_model_with_fp8_torchao_config(model, fp8_mode) @@ -1099,7 +1106,7 @@ def from_pretrained( # Find FP8, BnB 4bit, other mapped names old_model_name = model_name fp8_mode = None - restore_fp8_scales = load_in_fp8 != False + restore_fp8_scales = False if not use_exact_model_name: new_model_name = get_model_name( model_name, load_in_4bit = load_in_4bit, load_in_fp8 = load_in_fp8 @@ -1114,12 +1121,14 @@ def from_pretrained( load_in_16bit, ) model_name = _offline_quantize_to_fp8(model_name, fp8_mode, text_only = text_only) + restore_fp8_scales = True else: assert new_model_name is not None model_name = new_model_name # If mapper resolved to a pre-quantized FP8 model, disable # on-the-fly quantization to avoid double quantization if load_in_fp8 != False and new_model_name != old_model_name: + restore_fp8_scales = True load_in_fp8 = False # Check if pre-quantized models are allowed @@ -1480,7 +1489,11 @@ def _dispatch_diffusion(): # Check base model again for PEFT model_name = peft_config.base_model_name_or_path if not use_exact_model_name: - model_name = get_model_name(model_name, load_in_4bit) + model_name = get_model_name( + model_name, + load_in_4bit = load_in_4bit, + load_in_fp8 = load_in_fp8, + ) # Check if pre-quantized models are allowed # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64) if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith( @@ -1752,6 +1765,11 @@ def _dispatch_diffusion(): token = token, revision = revision if not is_peft else None, local_files_only = local_files_only, + subfolder = kwargs.get("subfolder"), + variant = kwargs.get("variant"), + use_safetensors = kwargs.get("use_safetensors"), + cache_dir = kwargs.get("cache_dir"), + force_download = kwargs.get("force_download", False), ) _tag_model_with_fp8_torchao_config(model, fp8_mode) diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 4c0761e57d0f..9cc933896847 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -14,6 +14,7 @@ from ..device_type import DEVICE_TYPE_TORCH import importlib +import json import logging import os import torch @@ -344,7 +345,7 @@ def _offline_quantize_to_fp8( **load_kwargs, ) tokenizer = auto_processor.from_pretrained(model_name) - model.save_pretrained(new_model_name, safe_serialization = False) + model.save_pretrained(new_model_name, safe_serialization = True) del model for _ in range(2): torch.cuda.empty_cache() @@ -412,6 +413,13 @@ def _model_has_real_fp8_modules(model): return False +_FP8_SCALE_SUFFIXES = ( + ("weight_scale_inv", ".weight_scale_inv"), + ("gate_up_proj_scale_inv", ".gate_up_proj_scale_inv"), + ("down_proj_scale_inv", ".down_proj_scale_inv"), +) + + def _resolve_weight_scale_inv_candidates(module_name, named_modules): if module_name in named_modules: return module_name @@ -422,7 +430,82 @@ def _resolve_weight_scale_inv_candidates(module_name, named_modules): return None -def _find_fp8_scale_inv_tensors(model_dir, model_name, revision = None, token = None, local_files_only = False): +def _variant_file_names(variant): + if variant: + return [ + f"model.{variant}.safetensors", + f"pytorch_model.{variant}.safetensors", + ] + return [ + "model.safetensors", + "pytorch_model.safetensors", + ] + + +def _variant_index_names(variant): + if variant: + return [ + f"model.{variant}.safetensors.index.json", + f"pytorch_model.{variant}.safetensors.index.json", + ] + return [ + "model.safetensors.index.json", + "pytorch_model.safetensors.index.json", + ] + + +def _resolve_fp8_scale_safetensors_files(model_dir, variant = None, use_safetensors = None): + if use_safetensors is False: + return [] + + for index_name in _variant_index_names(variant): + index_path = os.path.join(model_dir, index_name) + if not os.path.isfile(index_path): + continue + try: + with open(index_path, encoding = "utf-8") as fh: + weight_map = json.load(fh).get("weight_map", {}) + except Exception: + continue + selected = [] + for key, filename in weight_map.items(): + if not any(key.endswith(suffix) for _, suffix in _FP8_SCALE_SUFFIXES): + continue + full_path = os.path.join(model_dir, filename) + if filename.endswith(".safetensors") and os.path.isfile(full_path): + selected.append(full_path) + return sorted(set(selected)) + + for filename in _variant_file_names(variant): + full_path = os.path.join(model_dir, filename) + if os.path.isfile(full_path): + return [full_path] + + if variant is not None: + return [] + + safetensors_files = [ + os.path.join(model_dir, filename) + for filename in os.listdir(model_dir) + if filename.endswith(".safetensors") and os.path.isfile(os.path.join(model_dir, filename)) + ] + if len(safetensors_files) == 1: + return safetensors_files + return [] + + +def _find_fp8_scale_inv_tensors( + model_dir, + model_name, + revision = None, + token = None, + local_files_only = False, + subfolder = None, + variant = None, + use_safetensors = None, + cache_dir = None, + force_download = False, +): if not os.path.exists(model_dir): try: from huggingface_hub import snapshot_download @@ -435,6 +518,8 @@ def _find_fp8_scale_inv_tensors(model_dir, model_name, revision = None, token = revision = revision, token = token, local_files_only = local_files_only, + cache_dir = cache_dir, + force_download = force_download, ) except Exception: return [] @@ -442,42 +527,34 @@ def _find_fp8_scale_inv_tensors(model_dir, model_name, revision = None, token = if not os.path.isdir(model_dir): return [] + if subfolder: + model_dir = os.path.join(model_dir, subfolder) + if not os.path.isdir(model_dir): + return [] + try: from safetensors.torch import safe_open except Exception: return [] module_scales = {} - suffix = ".weight_scale_inv" - for filename in os.listdir(model_dir): - full_path = os.path.join(model_dir, filename) - if filename.endswith(".safetensors"): - try: - with safe_open(full_path, framework = "pt", device = "cpu") as f: - for key in f.keys(): + for full_path in _resolve_fp8_scale_safetensors_files( + model_dir, + variant = variant, + use_safetensors = use_safetensors, + ): + try: + with safe_open(full_path, framework = "pt", device = "cpu") as f: + for key in f.keys(): + for attr_name, suffix in _FP8_SCALE_SUFFIXES: if not key.endswith(suffix): continue module_name = key[: -len(suffix)] - module_scales[module_name] = f.get_tensor(key) - except Exception: - continue - continue - if not filename.endswith(".bin"): - continue - try: - state_dict = torch.load(full_path, map_location = "cpu") - if isinstance(state_dict, dict) and isinstance(state_dict.get("state_dict"), dict): - state_dict = state_dict["state_dict"] - if not isinstance(state_dict, dict): - continue - for key, value in state_dict.items(): - if not key.endswith(suffix) or not isinstance(value, torch.Tensor): - continue - module_name = key[: -len(suffix)] - module_scales[module_name] = value + module_scales[(module_name, attr_name)] = f.get_tensor(key) + break except Exception: continue - return list(module_scales.items()) + return [(module_name, attr_name, tensor) for (module_name, attr_name), tensor in module_scales.items()] def _restore_missing_fp8_weight_scale_inv( @@ -486,6 +563,9 @@ def _restore_missing_fp8_weight_scale_inv( token = None, revision = None, local_files_only = False, + subfolder = None, + variant = None, + use_safetensors = None, ): """Find checkpointed `.weight_scale_inv` tensors and restore missing runtime tensors on FP8 modules. @@ -501,43 +581,59 @@ def _restore_missing_fp8_weight_scale_inv( revision = revision, token = token, local_files_only = local_files_only, + subfolder = subfolder, + variant = variant, + use_safetensors = use_safetensors, ) if len(scaled_tensors) == 0: return 0, 0 restored = 0 skipped = 0 - for module_name, scale_tensor in scaled_tensors: + for module_name, attr_name, scale_tensor in scaled_tensors: target_name = _resolve_weight_scale_inv_candidates(module_name, name_to_module) if target_name is None: skipped += 1 continue module = name_to_module[target_name] - if not _is_real_fp8_owner(module): + if not _is_real_fp8_owner(module) and not hasattr(module, attr_name): skipped += 1 continue weight = getattr(module, "weight", None) - if not isinstance(weight, torch.Tensor): + reference = getattr(module, attr_name, None) + if not isinstance(reference, torch.Tensor) and attr_name == "weight_scale_inv": + reference = getattr(module, "weight_scale", None) + if not isinstance(weight, torch.Tensor) and not isinstance(reference, torch.Tensor): + skipped += 1 + continue + target_device = reference.device if isinstance(reference, torch.Tensor) else weight.device + if target_device.type == "meta": + skipped += 1 + continue + if isinstance(weight, torch.Tensor) and weight.device.type == "meta": + skipped += 1 + continue + if isinstance(reference, torch.Tensor) and reference.shape != scale_tensor.shape: + skipped += 1 + continue + if not isinstance(reference, torch.Tensor) and scale_tensor.numel() != 1: skipped += 1 continue - restored_scale = scale_tensor.to(device = weight.device) - reference = getattr(module, "weight_scale_inv", None) - if not isinstance(reference, torch.Tensor): - reference = getattr(module, "weight_scale", None) + restored_scale = scale_tensor.to(device = target_device) if isinstance(reference, torch.Tensor): try: restored_scale = restored_scale.to(dtype = reference.dtype) except Exception: pass - if "weight_scale_inv" in module._buffers: - module._buffers["weight_scale_inv"] = restored_scale + if attr_name in module._buffers: + module._buffers[attr_name] = restored_scale else: - if hasattr(module, "weight_scale_inv"): - delattr(module, "weight_scale_inv") - module.register_buffer("weight_scale_inv", restored_scale) + if hasattr(module, attr_name): + delattr(module, attr_name) + module.register_buffer(attr_name, restored_scale) restored += 1 return restored, skipped From 7022b7063b94ebf5e0c0c89f8109920e91ab7573 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 06:33:19 +0000 Subject: [PATCH 07/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_fp8_scale_inv_load.py | 48 ++++++++++++++++++++++++-------- unsloth/models/loader_utils.py | 11 ++++++-- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 96788de8b916..8c79a3f59df8 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -83,7 +83,11 @@ def _load_loader_utils(): return module -def _make_checkpoint(path, tensors, filename = "model.safetensors"): +def _make_checkpoint( + path, + tensors, + filename = "model.safetensors", +): full_path = os.path.join(path, filename) save_file(tensors, full_path) return path @@ -96,7 +100,12 @@ def _make_bin_checkpoint(path, tensors): class _Fp8Owner(torch.nn.Module): - def __init__(self, weight, quant_method = "fp8", weight_scale = None): + def __init__( + self, + weight, + quant_method = "fp8", + weight_scale = None, + ): super().__init__() self.weight = torch.nn.Parameter(weight) self.quant_method = quant_method @@ -227,7 +236,11 @@ def test_skips_bin_checkpoint_without_torch_load(monkeypatch): checkpoint_dir, {"fp8.weight_scale_inv": torch.tensor([2.5], dtype = torch.float32)}, ) - monkeypatch.setattr(loader_utils.torch, "load", lambda *args, **kwargs: pytest.fail("torch.load should not run")) + monkeypatch.setattr( + loader_utils.torch, + "load", + lambda *args, **kwargs: pytest.fail("torch.load should not run"), + ) restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( model, model_name = checkpoint_dir, @@ -319,6 +332,7 @@ def test_safetensors_index_opens_only_scale_shards(monkeypatch): } } (Path(checkpoint_dir) / "model.safetensors.index.json").write_text(json.dumps(index)) + def recording_safe_open(path, *args, **kwargs): opened.append(Path(path).name) return original_safe_open(path, *args, **kwargs) @@ -339,7 +353,9 @@ def recording_safe_open(path, *args, **kwargs): def test_skips_meta_device_scale_restore(): loader_utils = _load_loader_utils() model = torch.nn.Module() - model.fp8 = _Fp8Owner(weight = torch.empty(4, 4, device = "meta"), weight_scale = torch.empty(1, device = "meta")) + model.fp8 = _Fp8Owner( + weight = torch.empty(4, 4, device = "meta"), weight_scale = torch.empty(1, device = "meta") + ) with tempfile.TemporaryDirectory() as checkpoint_dir: _make_checkpoint( @@ -409,7 +425,9 @@ def test_restores_fp8_expert_scale_tensors(): assert restored == 2 assert skipped == 0 - assert torch.equal(model.expert.gate_up_proj_scale_inv, torch.full((2,), 2.0, dtype = torch.float16)) + assert torch.equal( + model.expert.gate_up_proj_scale_inv, torch.full((2,), 2.0, dtype = torch.float16) + ) assert torch.equal(model.expert.down_proj_scale_inv, torch.full((2,), 3.0, dtype = torch.float16)) @@ -453,7 +471,10 @@ def test_loader_restores_fp8_scales_with_base_revision_for_peft(): for node in ast.walk(tree): if not isinstance(node, ast.Call): continue - if not isinstance(node.func, ast.Name) or node.func.id != "_restore_missing_fp8_weight_scale_inv": + if ( + not isinstance(node.func, ast.Name) + or node.func.id != "_restore_missing_fp8_weight_scale_inv" + ): continue revision_kw = next((kw for kw in node.keywords if kw.arg == "revision"), None) @@ -484,7 +505,10 @@ def test_loader_passes_selected_artifact_knobs_to_fp8_restore(): for node in ast.walk(tree): if not isinstance(node, ast.Call): continue - if not isinstance(node.func, ast.Name) or node.func.id != "_restore_missing_fp8_weight_scale_inv": + if ( + not isinstance(node.func, ast.Name) + or node.func.id != "_restore_missing_fp8_weight_scale_inv" + ): continue keyword_names = {kw.arg for kw in node.keywords} if {"subfolder", "variant", "use_safetensors"}.issubset(keyword_names): @@ -504,7 +528,10 @@ def test_fastmodel_peft_base_mapping_forwards_load_in_fp8(): for child in ast.walk(node): if not isinstance(child, ast.Assign): continue - if not any(isinstance(target, ast.Name) and target.id == "model_name" for target in child.targets): + if not any( + isinstance(target, ast.Name) and target.id == "model_name" + for target in child.targets + ): continue value = child.value if not isinstance(value, ast.Call): @@ -559,10 +586,7 @@ def test_fp8_probe_uses_absolute_optional_import_and_narrow_missing_module_guard if not isinstance(child, ast.Try): continue - imports = [ - stmt for stmt in child.body - if isinstance(stmt, ast.ImportFrom) - ] + imports = [stmt for stmt in child.body if isinstance(stmt, ast.ImportFrom)] handlers = child.handlers if not imports or not handlers: continue diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 9cc933896847..d68c855f3f72 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -454,7 +454,11 @@ def _variant_index_names(variant): ] -def _resolve_fp8_scale_safetensors_files(model_dir, variant = None, use_safetensors = None): +def _resolve_fp8_scale_safetensors_files( + model_dir, + variant = None, + use_safetensors = None, +): if use_safetensors is False: return [] @@ -554,7 +558,10 @@ def _find_fp8_scale_inv_tensors( break except Exception: continue - return [(module_name, attr_name, tensor) for (module_name, attr_name), tensor in module_scales.items()] + return [ + (module_name, attr_name, tensor) + for (module_name, attr_name), tensor in module_scales.items() + ] def _restore_missing_fp8_weight_scale_inv( From 9d8cdee31e17208cb21f810e1b5bddee3635b76d Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Tue, 7 Jul 2026 02:48:14 -0400 Subject: [PATCH 08/35] fix: restore selected FP8 scale variants without placeholders --- tests/test_fp8_scale_inv_load.py | 72 +++++++++++++++++++++++++++++++- unsloth/models/loader_utils.py | 8 ++-- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 8c79a3f59df8..f80f4c73afd6 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -397,6 +397,70 @@ def test_skips_shape_incompatible_local_scale(): assert not hasattr(model.fp8, "weight_scale_inv") +def test_restores_multielement_scale_without_local_placeholder(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.ones((2, 1), dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert torch.equal(model.fp8.weight_scale_inv, torch.ones((2, 1), dtype = torch.float16)) + + +def test_restores_fbgemm_weight_scale_tensor(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale": torch.full((2, 1), 2.0, dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert torch.equal(model.fp8.weight_scale, torch.full((2, 1), 2.0, dtype = torch.float16)) + + +def test_restore_accepts_cache_download_selection_kwargs(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.25], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + cache_dir = checkpoint_dir, + force_download = False, + ) + + assert restored == 1 + assert skipped == 0 + + class _Fp8Expert(torch.nn.Module): def __init__(self): super().__init__() @@ -511,7 +575,13 @@ def test_loader_passes_selected_artifact_knobs_to_fp8_restore(): ): continue keyword_names = {kw.arg for kw in node.keywords} - if {"subfolder", "variant", "use_safetensors"}.issubset(keyword_names): + if { + "subfolder", + "variant", + "use_safetensors", + "cache_dir", + "force_download", + }.issubset(keyword_names): hit_count += 1 assert hit_count == 2 diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index d68c855f3f72..a9e3fa3b181d 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -414,6 +414,7 @@ def _model_has_real_fp8_modules(model): _FP8_SCALE_SUFFIXES = ( + ("weight_scale", ".weight_scale"), ("weight_scale_inv", ".weight_scale_inv"), ("gate_up_proj_scale_inv", ".gate_up_proj_scale_inv"), ("down_proj_scale_inv", ".down_proj_scale_inv"), @@ -573,6 +574,8 @@ def _restore_missing_fp8_weight_scale_inv( subfolder = None, variant = None, use_safetensors = None, + cache_dir = None, + force_download = False, ): """Find checkpointed `.weight_scale_inv` tensors and restore missing runtime tensors on FP8 modules. @@ -591,6 +594,8 @@ def _restore_missing_fp8_weight_scale_inv( subfolder = subfolder, variant = variant, use_safetensors = use_safetensors, + cache_dir = cache_dir, + force_download = force_download, ) if len(scaled_tensors) == 0: return 0, 0 @@ -625,9 +630,6 @@ def _restore_missing_fp8_weight_scale_inv( if isinstance(reference, torch.Tensor) and reference.shape != scale_tensor.shape: skipped += 1 continue - if not isinstance(reference, torch.Tensor) and scale_tensor.numel() != 1: - skipped += 1 - continue restored_scale = scale_tensor.to(device = target_device) if isinstance(reference, torch.Tensor): From 2c560a6c6c10fa540d963d772fe4f9f2d4b0514e Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Tue, 7 Jul 2026 02:52:42 -0400 Subject: [PATCH 09/35] fix: keep FP8 scale restore shard-aware --- tests/test_fp8_scale_inv_load.py | 48 ++++++++++++++++++++++++++++++-- unsloth/models/loader_utils.py | 12 +++++++- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index f80f4c73afd6..dd5c27235585 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -418,6 +418,27 @@ def test_restores_multielement_scale_without_local_placeholder(): assert torch.equal(model.fp8.weight_scale_inv, torch.ones((2, 1), dtype = torch.float16)) +def test_skips_oversized_scale_without_local_placeholder(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.ones((8, 1), dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 0 + assert skipped == 1 + assert not hasattr(model.fp8, "weight_scale_inv") + + def test_restores_fbgemm_weight_scale_tensor(): loader_utils = _load_loader_utils() model = torch.nn.Module() @@ -587,6 +608,29 @@ def test_loader_passes_selected_artifact_knobs_to_fp8_restore(): assert hit_count == 2 +def test_fp8_owner_probe_tolerates_optional_kernel_import_errors(): + tree = ast.parse(LOADER_UTILS.read_text()) + + for node in ast.walk(tree): + if not isinstance(node, ast.Try): + continue + if not any( + isinstance(child, ast.ImportFrom) + and child.module == "unsloth.kernels.fp8" + for child in node.body + ): + continue + assert any( + handler.type is not None + and isinstance(handler.type, ast.Name) + and handler.type.id == "Exception" + for handler in node.handlers + ) + return + + raise AssertionError("FP8 owner probe import guard not found") + + def test_fastmodel_peft_base_mapping_forwards_load_in_fp8(): tree = ast.parse(LOADER.read_text()) @@ -646,7 +690,7 @@ def test_model_has_real_fp8_modules_detects_nested_fp8_layers(): assert loader_utils._model_has_real_fp8_modules(model) is True -def test_fp8_probe_uses_absolute_optional_import_and_narrow_missing_module_guard(): +def test_fp8_probe_uses_absolute_optional_import_and_broad_nonfatal_guard(): tree = ast.parse(LOADER_UTILS.read_text()) for node in ast.walk(tree): @@ -665,7 +709,7 @@ def test_fp8_probe_uses_absolute_optional_import_and_narrow_missing_module_guard handler = handlers[0] assert import_stmt.module == "unsloth.kernels.fp8" assert isinstance(handler.type, ast.Name) - assert handler.type.id == "ModuleNotFoundError" + assert handler.type.id == "Exception" return pytest.fail("_is_real_fp8_owner no longer protects the optional fp8 kernel import.") diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index a9e3fa3b181d..df866eaa155a 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -393,7 +393,7 @@ def _is_real_fp8_owner(module): try: from unsloth.kernels.fp8 import FbgemmFp8Linear, FP8Linear - except ModuleNotFoundError as exc: + except Exception as exc: logger.debug("Optional fp8 kernels not available: %s", exc) FbgemmFp8Linear, FP8Linear = None, None if FbgemmFp8Linear is not None and isinstance(module, FbgemmFp8Linear): @@ -630,6 +630,16 @@ def _restore_missing_fp8_weight_scale_inv( if isinstance(reference, torch.Tensor) and reference.shape != scale_tensor.shape: skipped += 1 continue + if ( + not isinstance(reference, torch.Tensor) + and isinstance(weight, torch.Tensor) + and scale_tensor.numel() != 1 + and scale_tensor.ndim > 0 + and weight.ndim > 0 + and scale_tensor.shape[0] > weight.shape[0] + ): + skipped += 1 + continue restored_scale = scale_tensor.to(device = target_device) if isinstance(reference, torch.Tensor): From 7eb0d4732b6a93305402ff27635c0ce778b5c14b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 06:53:12 +0000 Subject: [PATCH 10/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_fp8_scale_inv_load.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index dd5c27235585..394dc16ccaf5 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -615,8 +615,7 @@ def test_fp8_owner_probe_tolerates_optional_kernel_import_errors(): if not isinstance(node, ast.Try): continue if not any( - isinstance(child, ast.ImportFrom) - and child.module == "unsloth.kernels.fp8" + isinstance(child, ast.ImportFrom) and child.module == "unsloth.kernels.fp8" for child in node.body ): continue From c619ad400b1657f54609e8eabc789477bfea0f37 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Tue, 7 Jul 2026 03:21:18 -0400 Subject: [PATCH 11/35] fix: align FP8 restore with selected safetensors caches --- tests/test_fp8_scale_inv_load.py | 93 ++++++++++++++++++++++++++++++++ unsloth/models/loader.py | 2 + unsloth/models/loader_utils.py | 6 ++- 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 394dc16ccaf5..5cd1f11a0ca1 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -350,6 +350,40 @@ def recording_safe_open(path, *args, **kwargs): assert opened == ["scale.safetensors"] +def test_variant_safetensors_index_uses_transformers_name(monkeypatch): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _PackedFp8Owner() + opened = [] + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([6.0], dtype = torch.float32)}, + filename = "scale.fp8.safetensors", + ) + index = {"weight_map": {"fp8.weight_scale_inv": "scale.fp8.safetensors"}} + (Path(checkpoint_dir) / "model.safetensors.index.fp8.json").write_text(json.dumps(index)) + + def recording_safe_open(path, *args, **kwargs): + opened.append(Path(path).name) + return original_safe_open(path, *args, **kwargs) + + original_safe_open = safetensors.torch.safe_open + monkeypatch.setattr(safetensors.torch, "safe_open", recording_safe_open) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + variant = "fp8", + ) + + assert restored == 1 + assert skipped == 0 + assert opened == ["scale.fp8.safetensors"] + assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([6.0], dtype = torch.float32)) + + def test_skips_meta_device_scale_restore(): loader_utils = _load_loader_utils() model = torch.nn.Module() @@ -487,6 +521,8 @@ def __init__(self): super().__init__() self.gate_up_proj_scale_inv = torch.ones(2, dtype = torch.float16) self.down_proj_scale_inv = torch.ones(2, dtype = torch.float16) + self.gate_up_proj_scale = torch.ones(2, dtype = torch.float16) + self.down_proj_scale = torch.ones(2, dtype = torch.float16) def test_restores_fp8_expert_scale_tensors(): @@ -516,6 +552,31 @@ def test_restores_fp8_expert_scale_tensors(): assert torch.equal(model.expert.down_proj_scale_inv, torch.full((2,), 3.0, dtype = torch.float16)) +def test_restores_fbgemm_expert_scale_tensors(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.expert = _Fp8Expert() + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + { + "expert.gate_up_proj_scale": torch.full((2,), 4.0, dtype = torch.float32), + "expert.down_proj_scale": torch.full((2,), 5.0, dtype = torch.float32), + }, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 2 + assert skipped == 0 + assert torch.equal(model.expert.gate_up_proj_scale, torch.full((2,), 4.0, dtype = torch.float16)) + assert torch.equal(model.expert.down_proj_scale, torch.full((2,), 5.0, dtype = torch.float16)) + + def test_loader_detects_direct_or_requested_fp8_restore_paths(): tree = ast.parse(LOADER.read_text()) hit_count = 0 @@ -674,6 +735,38 @@ def test_offline_fp8_cache_uses_safe_serialization(): pytest.fail("_offline_quantize_to_fp8 must save safetensors caches.") +def test_offline_fp8_cache_forces_safetensors_loads(): + tree = ast.parse(LOADER.read_text()) + hit_lines = set() + + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + if not any( + isinstance(child, ast.Call) + and isinstance(child.func, ast.Name) + and child.func.id == "_offline_quantize_to_fp8" + for child in ast.walk(node) + ): + continue + for child in ast.walk(node): + if not isinstance(child, ast.Assign): + continue + for target in child.targets: + if ( + isinstance(target, ast.Subscript) + and isinstance(target.value, ast.Name) + and target.value.id == "kwargs" + and isinstance(target.slice, ast.Constant) + and target.slice.value == "use_safetensors" + and isinstance(child.value, ast.Constant) + and child.value.value is True + ): + hit_lines.add(child.lineno) + + assert len(hit_lines) == 2 + + def test_fp8_probe_skips_missing_optional_kernel_module(): loader_utils = _load_loader_utils() module = torch.nn.Linear(4, 4, bias = False) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 3302383d1640..708e655c37ab 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -458,6 +458,7 @@ def from_pretrained( load_in_16bit, ) model_name = _offline_quantize_to_fp8(model_name, fp8_mode, text_only = text_only) + kwargs["use_safetensors"] = True restore_fp8_scales = True else: assert new_model_name is not None @@ -1121,6 +1122,7 @@ def from_pretrained( load_in_16bit, ) model_name = _offline_quantize_to_fp8(model_name, fp8_mode, text_only = text_only) + kwargs["use_safetensors"] = True restore_fp8_scales = True else: assert new_model_name is not None diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index df866eaa155a..a084deb59f09 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -416,7 +416,9 @@ def _model_has_real_fp8_modules(model): _FP8_SCALE_SUFFIXES = ( ("weight_scale", ".weight_scale"), ("weight_scale_inv", ".weight_scale_inv"), + ("gate_up_proj_scale", ".gate_up_proj_scale"), ("gate_up_proj_scale_inv", ".gate_up_proj_scale_inv"), + ("down_proj_scale", ".down_proj_scale"), ("down_proj_scale_inv", ".down_proj_scale_inv"), ) @@ -446,8 +448,8 @@ def _variant_file_names(variant): def _variant_index_names(variant): if variant: return [ - f"model.{variant}.safetensors.index.json", - f"pytorch_model.{variant}.safetensors.index.json", + f"model.safetensors.index.{variant}.json", + f"pytorch_model.safetensors.index.{variant}.json", ] return [ "model.safetensors.index.json", From 8f43bb8fec5a5c37286ac0a4a3a30add8b1f60f8 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Tue, 7 Jul 2026 03:38:51 -0400 Subject: [PATCH 12/35] fix: keep FP8 scale repair aligned with safe caches --- tests/test_fp8_scale_inv_load.py | 107 +++++++++++++++++++++++++++++++ unsloth/models/loader_utils.py | 20 +++++- 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 5cd1f11a0ca1..be097f8ee303 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -65,6 +65,7 @@ def _install_loader_utils_stubs(): model_utils = types.ModuleType("unsloth.models._utils") model_utils.TorchAOConfig = type("TorchAOConfig", (), {}) + model_utils._apply_text_only_key_mapping = lambda *args, **kwargs: None sys.modules["unsloth.models._utils"] = model_utils zoo_utils = types.ModuleType("unsloth_zoo.utils") @@ -431,6 +432,33 @@ def test_skips_shape_incompatible_local_scale(): assert not hasattr(model.fp8, "weight_scale_inv") +def test_reshapes_same_numel_local_scale_placeholder(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner( + weight = torch.randn(4, 4, dtype = torch.float16), + weight_scale = torch.ones((1, 4), dtype = torch.float16), + ) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.arange(4, dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert model.fp8.weight_scale_inv.shape == (1, 4) + assert torch.equal( + model.fp8.weight_scale_inv, torch.arange(4, dtype = torch.float16).reshape(1, 4) + ) + + def test_restores_multielement_scale_without_local_placeholder(): loader_utils = _load_loader_utils() model = torch.nn.Module() @@ -519,6 +547,8 @@ def test_restore_accepts_cache_download_selection_kwargs(): class _Fp8Expert(torch.nn.Module): def __init__(self): super().__init__() + self.up_proj_scale_inv = torch.ones(2, dtype = torch.float16) + self.up_proj_scale = torch.ones(2, dtype = torch.float16) self.gate_up_proj_scale_inv = torch.ones(2, dtype = torch.float16) self.down_proj_scale_inv = torch.ones(2, dtype = torch.float16) self.gate_up_proj_scale = torch.ones(2, dtype = torch.float16) @@ -577,6 +607,34 @@ def test_restores_fbgemm_expert_scale_tensors(): assert torch.equal(model.expert.down_proj_scale, torch.full((2,), 5.0, dtype = torch.float16)) +def test_restores_fp8_scale_alias_tensors(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) + model.expert = _Fp8Expert() + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + { + "fp8.scale": torch.full((2, 1), 6.0, dtype = torch.float32), + "expert.up_proj_scale": torch.full((2,), 7.0, dtype = torch.float32), + "expert.up_proj_scale_inv": torch.full((2,), 8.0, dtype = torch.float32), + }, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 3 + assert skipped == 0 + assert torch.equal(model.fp8.weight_scale, torch.full((2, 1), 6.0, dtype = torch.float16)) + assert torch.equal(model.expert.up_proj_scale, torch.full((2,), 7.0, dtype = torch.float16)) + assert torch.equal(model.expert.up_proj_scale_inv, torch.full((2,), 8.0, dtype = torch.float16)) + + def test_loader_detects_direct_or_requested_fp8_restore_paths(): tree = ast.parse(LOADER.read_text()) hit_count = 0 @@ -735,6 +793,55 @@ def test_offline_fp8_cache_uses_safe_serialization(): pytest.fail("_offline_quantize_to_fp8 must save safetensors caches.") +def test_offline_fp8_cache_regenerates_bin_only_cache(monkeypatch): + loader_utils = _load_loader_utils() + + class DummyConfig: + architectures = ["DummyForCausalLM"] + + class DummyModel: + def save_pretrained(self, path, safe_serialization = False): + assert safe_serialization is True + os.makedirs(path, exist_ok = True) + Path(path, "model.safetensors").write_text("safe") + + class DummyTokenizer: + def save_pretrained(self, path): + Path(path, "tokenizer.json").write_text("{}") + + class DummyAutoModel: + @staticmethod + def from_pretrained(*args, **kwargs): + return DummyModel() + + class DummyTokenizerLoader: + @staticmethod + def from_pretrained(*args, **kwargs): + return DummyTokenizer() + + import transformers + + monkeypatch.setattr(transformers.AutoConfig, "from_pretrained", lambda *args, **kwargs: DummyConfig()) + monkeypatch.setattr(transformers, "AutoModelForCausalLM", DummyAutoModel) + monkeypatch.setattr(transformers, "AutoModelForImageTextToText", DummyAutoModel) + monkeypatch.setattr(transformers, "AutoTokenizer", DummyTokenizerLoader) + monkeypatch.setattr(transformers, "AutoProcessor", DummyTokenizerLoader) + monkeypatch.setattr(transformers, "TorchAoConfig", lambda config: config) + monkeypatch.setattr(loader_utils, "_get_torchao_fp8_config", lambda mode: {"mode": mode}) + monkeypatch.setattr(loader_utils.torch.cuda, "empty_cache", lambda: None) + + with tempfile.TemporaryDirectory() as temp_dir: + monkeypatch.setattr(loader_utils.tempfile, "gettempdir", lambda: temp_dir) + stale_cache = Path(temp_dir) / "model-fp8-row" + stale_cache.mkdir() + Path(stale_cache, "pytorch_model.bin").write_text("old") + + result = loader_utils._offline_quantize_to_fp8("org/model", "row") + assert result == str(stale_cache) + assert not Path(result, "pytorch_model.bin").exists() + assert Path(result, "model.safetensors").exists() + + def test_offline_fp8_cache_forces_safetensors_loads(): tree = ast.parse(LOADER.read_text()) hit_lines = set() diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index a084deb59f09..e0d23146f7f8 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -17,6 +17,7 @@ import json import logging import os +import shutil import torch import re import tempfile @@ -328,6 +329,11 @@ def _offline_quantize_to_fp8( new_model_name = os.path.join(temp_dir, cache_name) print(f"Unsloth: Quantizing '{model_name}' to fp8, using model_name='{new_model_name}' instead") + if os.path.isdir(new_model_name) and not any( + filename.endswith(".safetensors") for filename in os.listdir(new_model_name) + ): + shutil.rmtree(new_model_name) + if not os.path.isdir(new_model_name): from ._utils import _apply_text_only_key_mapping @@ -414,8 +420,13 @@ def _model_has_real_fp8_modules(model): _FP8_SCALE_SUFFIXES = ( + ("weight_scale", ".scale"), ("weight_scale", ".weight_scale"), ("weight_scale_inv", ".weight_scale_inv"), + ("gate_proj_scale", ".gate_proj_scale"), + ("gate_proj_scale_inv", ".gate_proj_scale_inv"), + ("up_proj_scale", ".up_proj_scale"), + ("up_proj_scale_inv", ".up_proj_scale_inv"), ("gate_up_proj_scale", ".gate_up_proj_scale"), ("gate_up_proj_scale_inv", ".gate_up_proj_scale_inv"), ("down_proj_scale", ".down_proj_scale"), @@ -629,9 +640,12 @@ def _restore_missing_fp8_weight_scale_inv( if isinstance(weight, torch.Tensor) and weight.device.type == "meta": skipped += 1 continue - if isinstance(reference, torch.Tensor) and reference.shape != scale_tensor.shape: - skipped += 1 - continue + target_shape = reference.shape if isinstance(reference, torch.Tensor) else None + if target_shape is not None and target_shape != scale_tensor.shape: + if reference.numel() != scale_tensor.numel(): + skipped += 1 + continue + scale_tensor = scale_tensor.reshape(target_shape) if ( not isinstance(reference, torch.Tensor) and isinstance(weight, torch.Tensor) From 2331a14cc0d79d4badca60401566781f8b09136e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 07:39:27 +0000 Subject: [PATCH 13/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_fp8_scale_inv_load.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index be097f8ee303..30d854f434fc 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -800,7 +800,11 @@ class DummyConfig: architectures = ["DummyForCausalLM"] class DummyModel: - def save_pretrained(self, path, safe_serialization = False): + def save_pretrained( + self, + path, + safe_serialization = False, + ): assert safe_serialization is True os.makedirs(path, exist_ok = True) Path(path, "model.safetensors").write_text("safe") @@ -821,7 +825,9 @@ def from_pretrained(*args, **kwargs): import transformers - monkeypatch.setattr(transformers.AutoConfig, "from_pretrained", lambda *args, **kwargs: DummyConfig()) + monkeypatch.setattr( + transformers.AutoConfig, "from_pretrained", lambda *args, **kwargs: DummyConfig() + ) monkeypatch.setattr(transformers, "AutoModelForCausalLM", DummyAutoModel) monkeypatch.setattr(transformers, "AutoModelForImageTextToText", DummyAutoModel) monkeypatch.setattr(transformers, "AutoTokenizer", DummyTokenizerLoader) From 1289285e0eadf28e78d9d99844f1d8c38e6300cb Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Tue, 7 Jul 2026 03:59:34 -0400 Subject: [PATCH 14/35] fix: preserve FP8 remap intent through PEFT --- tests/test_fp8_scale_inv_load.py | 63 +++++++++++++++++++++++++++++++- unsloth/models/loader.py | 3 ++ unsloth/models/loader_utils.py | 39 +++++++++++++++----- 3 files changed, 95 insertions(+), 10 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 30d854f434fc..6fbe3a55c8aa 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -555,6 +555,12 @@ def __init__(self): self.down_proj_scale = torch.ones(2, dtype = torch.float16) +class _WeightlessFp8Expert(torch.nn.Module): + def __init__(self): + super().__init__() + self.quant_method = "fp8" + + def test_restores_fp8_expert_scale_tensors(): loader_utils = _load_loader_utils() model = torch.nn.Module() @@ -607,6 +613,33 @@ def test_restores_fbgemm_expert_scale_tensors(): assert torch.equal(model.expert.down_proj_scale, torch.full((2,), 5.0, dtype = torch.float16)) +def test_restores_missing_weightless_fp8_expert_scale_tensors(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.expert = _WeightlessFp8Expert() + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + { + "expert.gate_up_proj_scale_inv": torch.full((2,), 9.0, dtype = torch.float32), + "expert.down_proj_scale_inv": torch.full((2,), 10.0, dtype = torch.float32), + }, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 2 + assert skipped == 0 + assert torch.equal( + model.expert.gate_up_proj_scale_inv, torch.full((2,), 9.0, dtype = torch.float32) + ) + assert torch.equal(model.expert.down_proj_scale_inv, torch.full((2,), 10.0, dtype = torch.float32)) + + def test_restores_fp8_scale_alias_tensors(): loader_utils = _load_loader_utils() model = torch.nn.Module() @@ -630,7 +663,7 @@ def test_restores_fp8_scale_alias_tensors(): assert restored == 3 assert skipped == 0 - assert torch.equal(model.fp8.weight_scale, torch.full((2, 1), 6.0, dtype = torch.float16)) + assert torch.equal(model.fp8.weight_scale_inv, torch.full((2, 1), 6.0, dtype = torch.float16)) assert torch.equal(model.expert.up_proj_scale, torch.full((2,), 7.0, dtype = torch.float16)) assert torch.equal(model.expert.up_proj_scale_inv, torch.full((2,), 8.0, dtype = torch.float16)) @@ -777,6 +810,34 @@ def test_fastmodel_peft_base_mapping_forwards_load_in_fp8(): pytest.fail("FastModel PEFT base mapping must forward load_in_fp8 to get_model_name.") +def test_fastmodel_peft_base_mapping_clears_fp8_after_remap(): + tree = ast.parse(LOADER.read_text()) + + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + if not isinstance(node.test, ast.Name) or node.test.id != "is_peft": + continue + has_old_model_name = False + has_clear = False + for child in ast.walk(node): + if isinstance(child, ast.Assign): + if any( + isinstance(target, ast.Name) and target.id == "old_model_name" + for target in child.targets + ): + has_old_model_name = True + if any( + isinstance(target, ast.Name) and target.id == "load_in_fp8" + for target in child.targets + ) and isinstance(child.value, ast.Constant) and child.value.value is False: + has_clear = True + if has_old_model_name and has_clear: + return + + pytest.fail("FastModel PEFT FP8 base remap must clear load_in_fp8 after selecting an FP8 sibling.") + + def test_offline_fp8_cache_uses_safe_serialization(): tree = ast.parse(LOADER_UTILS.read_text()) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 708e655c37ab..e067e93b2f18 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -1491,11 +1491,14 @@ def _dispatch_diffusion(): # Check base model again for PEFT model_name = peft_config.base_model_name_or_path if not use_exact_model_name: + old_model_name = model_name model_name = get_model_name( model_name, load_in_4bit = load_in_4bit, load_in_fp8 = load_in_fp8, ) + if load_in_fp8 != False and model_name != old_model_name: + load_in_fp8 = False # Check if pre-quantized models are allowed # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64) if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith( diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index e0d23146f7f8..a5c229097989 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -380,13 +380,8 @@ def _has_float8_dtype(dtype): def _is_real_fp8_owner(module): if not isinstance(module, torch.nn.Module): return False - weight = getattr(module, "weight", None) - if not isinstance(weight, torch.Tensor): - return False if getattr(module, "quant_method", None) in ["fp8", "fbgemm_fp8"]: return True - if getattr(weight, "quant_method", None) in ["fp8", "fbgemm_fp8"]: - return True config = getattr(module, "quantization_config", None) if isinstance(config, dict) and config.get("quant_method") in ["fp8", "fbgemm_fp8"]: @@ -394,9 +389,25 @@ def _is_real_fp8_owner(module): if getattr(config, "quant_method", None) in ["fp8", "fbgemm_fp8"]: return True - if _has_float8_dtype(getattr(weight, "dtype", None)): + if getattr(module, "quant_method", None) is None and "fp8" in type(module).__name__.lower(): return True + weight = getattr(module, "weight", None) + if isinstance(weight, torch.Tensor): + if getattr(weight, "quant_method", None) in ["fp8", "fbgemm_fp8"]: + return True + if _has_float8_dtype(getattr(weight, "dtype", None)): + return True + + for attr_name in ("gate_up_proj", "gate_proj", "up_proj", "down_proj"): + tensor = getattr(module, attr_name, None) + if not isinstance(tensor, torch.Tensor): + continue + if getattr(tensor, "quant_method", None) in ["fp8", "fbgemm_fp8"]: + return True + if _has_float8_dtype(getattr(tensor, "dtype", None)): + return True + try: from unsloth.kernels.fp8 import FbgemmFp8Linear, FP8Linear except Exception as exc: @@ -420,7 +431,7 @@ def _model_has_real_fp8_modules(model): _FP8_SCALE_SUFFIXES = ( - ("weight_scale", ".scale"), + ("weight_scale_inv", ".scale"), ("weight_scale", ".weight_scale"), ("weight_scale_inv", ".weight_scale_inv"), ("gate_proj_scale", ".gate_proj_scale"), @@ -630,10 +641,20 @@ def _restore_missing_fp8_weight_scale_inv( reference = getattr(module, attr_name, None) if not isinstance(reference, torch.Tensor) and attr_name == "weight_scale_inv": reference = getattr(module, "weight_scale", None) - if not isinstance(weight, torch.Tensor) and not isinstance(reference, torch.Tensor): + if ( + not _is_real_fp8_owner(module) + and not isinstance(weight, torch.Tensor) + and not isinstance(reference, torch.Tensor) + ): skipped += 1 continue - target_device = reference.device if isinstance(reference, torch.Tensor) else weight.device + target_device = ( + reference.device + if isinstance(reference, torch.Tensor) + else weight.device + if isinstance(weight, torch.Tensor) + else scale_tensor.device + ) if target_device.type == "meta": skipped += 1 continue From 96beef372628aaaa2c56005dd63d25dfbac8fe1f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:00:10 +0000 Subject: [PATCH 15/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_fp8_scale_inv_load.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 6fbe3a55c8aa..893a25ae402c 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -637,7 +637,9 @@ def test_restores_missing_weightless_fp8_expert_scale_tensors(): assert torch.equal( model.expert.gate_up_proj_scale_inv, torch.full((2,), 9.0, dtype = torch.float32) ) - assert torch.equal(model.expert.down_proj_scale_inv, torch.full((2,), 10.0, dtype = torch.float32)) + assert torch.equal( + model.expert.down_proj_scale_inv, torch.full((2,), 10.0, dtype = torch.float32) + ) def test_restores_fp8_scale_alias_tensors(): @@ -827,15 +829,21 @@ def test_fastmodel_peft_base_mapping_clears_fp8_after_remap(): for target in child.targets ): has_old_model_name = True - if any( - isinstance(target, ast.Name) and target.id == "load_in_fp8" - for target in child.targets - ) and isinstance(child.value, ast.Constant) and child.value.value is False: + if ( + any( + isinstance(target, ast.Name) and target.id == "load_in_fp8" + for target in child.targets + ) + and isinstance(child.value, ast.Constant) + and child.value.value is False + ): has_clear = True if has_old_model_name and has_clear: return - pytest.fail("FastModel PEFT FP8 base remap must clear load_in_fp8 after selecting an FP8 sibling.") + pytest.fail( + "FastModel PEFT FP8 base remap must clear load_in_fp8 after selecting an FP8 sibling." + ) def test_offline_fp8_cache_uses_safe_serialization(): From 0d783fdca250aeadbdb959ffc3b106950ff7d264 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 7 Jul 2026 09:46:54 +0000 Subject: [PATCH 16/35] Restore dropped FP8 expert scale on the projection device for PR #6749 --- tests/test_fp8_scale_inv_load.py | 37 ++++++++++++++++++++++++++++ unsloth/models/loader_utils.py | 42 ++++++++++++++++++++++++++------ 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 893a25ae402c..b8a9c525f27b 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -642,6 +642,43 @@ def test_restores_missing_weightless_fp8_expert_scale_tensors(): ) +class _ProjectionFp8Expert(torch.nn.Module): + """Weightless FP8 expert that keeps projections (like FbgemmFp8Llama4TextExperts) instead + of a top-level weight, with its scale placeholder dropped.""" + + def __init__(self, device): + super().__init__() + self.gate_up_proj = torch.zeros(4, 8, dtype = torch.float8_e4m3fn, device = device) + self.down_proj = torch.zeros(4, 8, dtype = torch.float8_e4m3fn, device = device) + + +@pytest.mark.skipif( + not torch.cuda.is_available(), + reason = "needs a non-CPU device to prove scale lands on the projection device", +) +def test_restores_projection_expert_scale_on_projection_device(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.expert = _ProjectionFp8Expert(device = "cuda:0") + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"expert.gate_up_proj_scale_inv": torch.full((4,), 2.0, dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + # Scanner opens on CPU; the restored buffer must follow the CUDA projection, not the + # checkpoint tensor, or the expert forward hits a device mismatch. #6749 + assert model.expert.gate_up_proj_scale_inv.device.type == "cuda" + + def test_restores_fp8_scale_alias_tensors(): loader_utils = _load_loader_utils() model = torch.nn.Module() diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index a5c229097989..38042be50a5d 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -589,6 +589,32 @@ def _find_fp8_scale_inv_tensors( ] +# Projection tensors kept by weightless FP8 experts (FP8Experts / +# FbgemmFp8Llama4TextExperts) in place of a top-level `weight`. +_FP8_PROJECTION_ATTRS = ("gate_up_proj", "gate_proj", "up_proj", "down_proj") + + +def _fp8_projection_device(module, attr_name): + """Device of the projection tensor a dropped-placeholder scale belongs to, else None. + + Weightless FP8 experts store projections like `gate_up_proj`/`down_proj` instead of a + top-level `weight`, so a restored scale must land on the projection's device. The scanner + opens safetensors on CPU, so falling back to the checkpoint tensor's device would register + a CPU buffer next to CUDA projections and the expert forward would hit a device mismatch. + """ + for suffix in ("_scale_inv", "_scale"): + if attr_name.endswith(suffix): + projection = getattr(module, attr_name[: -len(suffix)], None) + if isinstance(projection, torch.Tensor): + return projection.device + break + for proj_name in _FP8_PROJECTION_ATTRS: + projection = getattr(module, proj_name, None) + if isinstance(projection, torch.Tensor): + return projection.device + return None + + def _restore_missing_fp8_weight_scale_inv( model: torch.nn.Module, model_name: str, @@ -648,13 +674,15 @@ def _restore_missing_fp8_weight_scale_inv( ): skipped += 1 continue - target_device = ( - reference.device - if isinstance(reference, torch.Tensor) - else weight.device - if isinstance(weight, torch.Tensor) - else scale_tensor.device - ) + if isinstance(reference, torch.Tensor): + target_device = reference.device + elif isinstance(weight, torch.Tensor): + target_device = weight.device + else: + projection_device = _fp8_projection_device(module, attr_name) + target_device = ( + projection_device if projection_device is not None else scale_tensor.device + ) if target_device.type == "meta": skipped += 1 continue From 2d95984d156f9e11daa9363d82c7ccce3cfbb13b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 7 Jul 2026 09:51:15 +0000 Subject: [PATCH 17/35] Preserve the adapter repo name through the FP8 base remap in FastModel PEFT loads for PR #6749 --- tests/test_fp8_scale_inv_load.py | 68 +++++++++++++++++++++++--------- unsloth/models/loader.py | 7 +++- 2 files changed, 54 insertions(+), 21 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index b8a9c525f27b..3737397d0f37 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -857,32 +857,62 @@ def test_fastmodel_peft_base_mapping_clears_fp8_after_remap(): continue if not isinstance(node.test, ast.Name) or node.test.id != "is_peft": continue - has_old_model_name = False - has_clear = False for child in ast.walk(node): - if isinstance(child, ast.Assign): - if any( - isinstance(target, ast.Name) and target.id == "old_model_name" - for target in child.targets - ): - has_old_model_name = True - if ( - any( - isinstance(target, ast.Name) and target.id == "load_in_fp8" - for target in child.targets - ) - and isinstance(child.value, ast.Constant) - and child.value.value is False - ): - has_clear = True - if has_old_model_name and has_clear: - return + # Look for: if load_in_fp8 != False and model_name != : load_in_fp8 = False + if not isinstance(child, ast.If): + continue + if not (isinstance(child.test, ast.BoolOp) and isinstance(child.test.op, ast.And)): + continue + compares_model_name = any( + isinstance(value, ast.Compare) + and isinstance(value.left, ast.Name) + and value.left.id == "model_name" + for value in child.test.values + ) + clears_fp8 = any( + isinstance(stmt, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "load_in_fp8" + for target in stmt.targets + ) + and isinstance(stmt.value, ast.Constant) + and stmt.value.value is False + for stmt in child.body + ) + if compares_model_name and clears_fp8: + return pytest.fail( "FastModel PEFT FP8 base remap must clear load_in_fp8 after selecting an FP8 sibling." ) +def test_fastmodel_peft_preserves_adapter_name_for_peft_load(): + # The FP8 base remap must compare against a pre-remap base name, but that comparison must + # NOT reuse old_model_name: old_model_name still points at the adapter repo that + # PeftModel.from_pretrained loads, so clobbering it inside the is_peft block would make + # PEFT attach the base checkpoint instead of the adapter. #6749 + tree = ast.parse(LOADER.read_text()) + offending = [] + + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + if not isinstance(node.test, ast.Name) or node.test.id != "is_peft": + continue + for child in ast.walk(node): + if isinstance(child, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "old_model_name" + for target in child.targets + ): + offending.append(child.lineno) + + assert offending == [], ( + "old_model_name must not be reassigned inside an is_peft block; it is the adapter " + f"repo used by PeftModel.from_pretrained. Reassigned at lines {offending}." + ) + + def test_offline_fp8_cache_uses_safe_serialization(): tree = ast.parse(LOADER_UTILS.read_text()) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index e067e93b2f18..17ec3c8d728f 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -1491,13 +1491,16 @@ def _dispatch_diffusion(): # Check base model again for PEFT model_name = peft_config.base_model_name_or_path if not use_exact_model_name: - old_model_name = model_name + # Compare against a separate local, not old_model_name: old_model_name still + # holds the adapter repo that PeftModel.from_pretrained loads below, so reusing + # it here would attach the base checkpoint instead of the adapter. #6749 + base_before_remap = model_name model_name = get_model_name( model_name, load_in_4bit = load_in_4bit, load_in_fp8 = load_in_fp8, ) - if load_in_fp8 != False and model_name != old_model_name: + if load_in_fp8 != False and model_name != base_before_remap: load_in_fp8 = False # Check if pre-quantized models are allowed # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64) From b61f390ac9036e98f03cacf1ad78f8dff97b904e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 7 Jul 2026 13:32:16 +0000 Subject: [PATCH 18/35] Constrain FP8 scale-restore snapshot download to selected safetensors for PR #6749 --- tests/test_fp8_scale_inv_load.py | 68 ++++++++++++++++++++++++++++++++ unsloth/models/loader_utils.py | 27 +++++++++++++ 2 files changed, 95 insertions(+) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 3737397d0f37..66bc17961bef 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -351,6 +351,74 @@ def recording_safe_open(path, *args, **kwargs): assert opened == ["scale.safetensors"] +def test_remote_scale_restore_constrains_snapshot_download_patterns(monkeypatch): + # A remote FP8 scale restore must not re-pull the whole repo after the model already + # loaded: constrain snapshot_download to the index + selected safetensors, never .bin. #6749 + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _PackedFp8Owner() + recorded = {} + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.0], dtype = torch.float32)}, + filename = "scale.safetensors", + ) + import huggingface_hub + + def fake_snapshot_download(repo_id, **kwargs): + recorded["allow_patterns"] = kwargs.get("allow_patterns") + return checkpoint_dir + + monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = "unsloth/does-not-exist-fp8", + local_files_only = False, + ) + + assert restored == 1 + assert skipped == 0 + allow_patterns = recorded["allow_patterns"] + assert allow_patterns is not None + assert any(pattern.endswith(".safetensors") for pattern in allow_patterns) + assert all("bin" not in pattern for pattern in allow_patterns) + + +def test_remote_scale_restore_snapshot_patterns_respect_variant(monkeypatch): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _PackedFp8Owner() + recorded = {} + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([2.0], dtype = torch.float32)}, + filename = "model.fp8.safetensors", + ) + import huggingface_hub + + def fake_snapshot_download(repo_id, **kwargs): + recorded["allow_patterns"] = kwargs.get("allow_patterns") + return checkpoint_dir + + monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = "unsloth/does-not-exist-fp8", + local_files_only = False, + variant = "fp8", + ) + + assert restored == 1 + assert skipped == 0 + allow_patterns = recorded["allow_patterns"] + assert allow_patterns is not None + assert all(".fp8.safetensors" in p or ".index.fp8.json" in p for p in allow_patterns) + + def test_variant_safetensors_index_uses_transformers_name(monkeypatch): loader_utils = _load_loader_utils() model = torch.nn.Module() diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 38042be50a5d..3d0dadf8033e 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -523,6 +523,22 @@ def _resolve_fp8_scale_safetensors_files( return [] +def _fp8_scale_snapshot_allow_patterns(subfolder = None, variant = None, use_safetensors = None): + """Glob patterns limiting the scale-restore snapshot to the same safetensors artifact + from_pretrained selected. Without this, snapshot_download would pull the FULL repo + (.bin shards, alternate variants, large extras) after the model already loaded, which + exhausts disk/network and defeats the safetensors-only selection. #6749""" + if use_safetensors is False: + return [] + if variant: + patterns = [f"*.{variant}.safetensors", f"*.safetensors.index.{variant}.json"] + else: + patterns = ["*.safetensors", "*.safetensors.index.json"] + if subfolder: + patterns = [f"{subfolder}/{pattern}" for pattern in patterns] + return patterns + + def _find_fp8_scale_inv_tensors( model_dir, model_name, @@ -541,6 +557,16 @@ def _find_fp8_scale_inv_tensors( except Exception: return [] + # Only the index + selected safetensor shards are needed; never the whole repo. + allow_patterns = _fp8_scale_snapshot_allow_patterns( + subfolder = subfolder, + variant = variant, + use_safetensors = use_safetensors, + ) + if not allow_patterns: + # use_safetensors is False -> nothing to restore, so skip the download entirely. + return [] + try: model_dir = snapshot_download( model_name, @@ -549,6 +575,7 @@ def _find_fp8_scale_inv_tensors( local_files_only = local_files_only, cache_dir = cache_dir, force_download = force_download, + allow_patterns = allow_patterns, ) except Exception: return [] From 04008edcbe4dd9cd448a88242e63a00d57f82199 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:32:47 +0000 Subject: [PATCH 19/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/loader_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 3d0dadf8033e..87b377a68a67 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -523,7 +523,11 @@ def _resolve_fp8_scale_safetensors_files( return [] -def _fp8_scale_snapshot_allow_patterns(subfolder = None, variant = None, use_safetensors = None): +def _fp8_scale_snapshot_allow_patterns( + subfolder = None, + variant = None, + use_safetensors = None, +): """Glob patterns limiting the scale-restore snapshot to the same safetensors artifact from_pretrained selected. Without this, snapshot_download would pull the FULL repo (.bin shards, alternate variants, large extras) after the model already loaded, which From 508c4aea02d41a410875176dc568dd66ce28e5c2 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 7 Jul 2026 14:09:26 +0000 Subject: [PATCH 20/35] Cover sharded FP8 variant shards in scale-restore patterns and match floating-owner scale dtype for PR #6749 --- tests/test_fp8_scale_inv_load.py | 80 +++++++++++++++++++++++++++++++- unsloth/models/loader_utils.py | 23 ++++++++- 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 66bc17961bef..841c4711f493 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1,8 +1,10 @@ import ast +import fnmatch import importlib.util import json import os from pathlib import Path +import shutil import sys import tempfile import types @@ -132,6 +134,7 @@ def test_restores_missing_fp8_weight_scale_inv(): assert restored == 1 assert skipped == 0 + assert model.fp8.weight_scale_inv.dtype == torch.float16 assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([1.25], dtype = torch.float16)) assert "fp8.weight_scale_inv" in model.state_dict() assert torch.equal( @@ -416,7 +419,82 @@ def fake_snapshot_download(repo_id, **kwargs): assert skipped == 0 allow_patterns = recorded["allow_patterns"] assert allow_patterns is not None - assert all(".fp8.safetensors" in p or ".index.fp8.json" in p for p in allow_patterns) + # Every pattern must be variant-scoped (mentions the variant) and none may be the bare + # non-variant safetensors/index globs, so the snapshot stays limited to the selected variant. + assert all("fp8" in p for p in allow_patterns) + assert not any(p in ("*.safetensors", "*.safetensors.index.json") for p in allow_patterns) + + +def test_remote_scale_restore_snapshot_patterns_cover_sharded_variant_shards(monkeypatch): + # transformers names a sharded variant shard `model.-00001-of-00002.safetensors` + # (variant applied before the shard suffix). The scale-restore allow_patterns must cover that + # form, or snapshot_download pulls the variant index but none of its shards and no scales are + # restored - a silent regression for remote sharded variant FP8 checkpoints. #6749 + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _PackedFp8Owner() + + with tempfile.TemporaryDirectory() as remote_dir, tempfile.TemporaryDirectory() as snap_dir: + shard_name = "model.fp8-00001-of-00002.safetensors" + _make_checkpoint( + remote_dir, + {"fp8.weight_scale_inv": torch.tensor([7.0], dtype = torch.float32)}, + filename = shard_name, + ) + index = {"weight_map": {"fp8.weight_scale_inv": shard_name}} + (Path(remote_dir) / "model.safetensors.index.fp8.json").write_text(json.dumps(index)) + # A non-variant shard the variant patterns must NOT pull. + _make_checkpoint( + remote_dir, + {"fp8.weight_scale_inv": torch.tensor([1.0], dtype = torch.float32)}, + filename = "model-00001-of-00002.safetensors", + ) + import huggingface_hub + + def fake_snapshot_download(repo_id, **kwargs): + # Emulate the hub applying allow_patterns: only matching files land in the snapshot. + patterns = kwargs.get("allow_patterns") or ["*"] + for name in os.listdir(remote_dir): + if any(fnmatch.fnmatch(name, p) for p in patterns): + shutil.copy(os.path.join(remote_dir, name), os.path.join(snap_dir, name)) + return snap_dir + + monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = "unsloth/does-not-exist-fp8", + local_files_only = False, + variant = "fp8", + ) + assert not os.path.isfile(os.path.join(snap_dir, "model-00001-of-00002.safetensors")) + + assert restored == 1 + assert skipped == 0 + assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([7.0], dtype = torch.float32)) + + +def test_restored_floating_owner_scale_matches_weight_dtype(): + # A floating (fp16/bf16) FP8 owner with no scale placeholder has no reference dtype to match, + # so the restored buffer must mirror the weight dtype (like the placeholder path) rather than + # keep the checkpoint's fp32, which would disagree with the module dtype at matmul. #6749 + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.bfloat16)) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.25], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert model.fp8.weight_scale_inv.dtype == torch.bfloat16 def test_variant_safetensors_index_uses_transformers_name(monkeypatch): diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 87b377a68a67..141268a429ee 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -535,7 +535,15 @@ def _fp8_scale_snapshot_allow_patterns( if use_safetensors is False: return [] if variant: - patterns = [f"*.{variant}.safetensors", f"*.safetensors.index.{variant}.json"] + # transformers applies the variant BEFORE the shard suffix, so a sharded variant shard + # is `model.-00001-of-00002.safetensors` (matched by `*.-*.safetensors`), + # not `model..safetensors` (only the single-file form). Cover both, else + # snapshot_download fetches the variant index but none of its shards and no scales load. #6749 + patterns = [ + f"*.{variant}.safetensors", + f"*.{variant}-*.safetensors", + f"*.safetensors.index.{variant}.json", + ] else: patterns = ["*.safetensors", "*.safetensors.index.json"] if subfolder: @@ -743,6 +751,19 @@ def _restore_missing_fp8_weight_scale_inv( restored_scale = restored_scale.to(dtype = reference.dtype) except Exception: pass + elif ( + isinstance(weight, torch.Tensor) + and weight.is_floating_point() + and not _has_float8_dtype(weight.dtype) + ): + # Floating (fp16/bf16) FP8 owner whose scale placeholder was dropped: no reference to + # match, so mirror the weight dtype like the placeholder path does instead of leaving + # the buffer in the checkpoint's fp32. Packed (int8) and real float8 weights keep the + # checkpoint dtype. #6749 + try: + restored_scale = restored_scale.to(dtype = weight.dtype) + except Exception: + pass if attr_name in module._buffers: module._buffers[attr_name] = restored_scale else: From 727a2c3f8077ec5000b667f68383bd50e562ba14 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 7 Jul 2026 15:02:43 +0000 Subject: [PATCH 21/35] Narrow no-variant FP8 scale snapshot patterns to default artifacts for PR #6749 --- tests/test_fp8_scale_inv_load.py | 27 +++++++++++++++++++++++++++ unsloth/models/loader_utils.py | 14 +++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 841c4711f493..77feaf5e8db3 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -389,6 +389,33 @@ def fake_snapshot_download(repo_id, **kwargs): assert all("bin" not in pattern for pattern in allow_patterns) +def test_no_variant_snapshot_patterns_exclude_alternate_variants(): + # The no-variant patterns must cover the default single-file/sharded/index names the scanner + # opens, but NOT alternate variant safetensors a repo may also publish. #6749 + import fnmatch + + loader_utils = _load_loader_utils() + patterns = loader_utils._fp8_scale_snapshot_allow_patterns() + + def matched(name): + return any(fnmatch.fnmatch(name, p) for p in patterns) + + for name in ( + "model.safetensors", + "model-00001-of-00002.safetensors", + "pytorch_model-00001-of-00002.safetensors", + "model.safetensors.index.json", + ): + assert matched(name), name + for name in ( + "model.fp8.safetensors", + "model.fp8-00001-of-00002.safetensors", + "pytorch_model.bin", + "model-00001-of-00002.bin", + ): + assert not matched(name), name + + def test_remote_scale_restore_snapshot_patterns_respect_variant(monkeypatch): loader_utils = _load_loader_utils() model = torch.nn.Module() diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 141268a429ee..c388ffc0908d 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -545,7 +545,19 @@ def _fp8_scale_snapshot_allow_patterns( f"*.safetensors.index.{variant}.json", ] else: - patterns = ["*.safetensors", "*.safetensors.index.json"] + # Restrict to the default (non-variant) transformers artifact names the scanner opens + # (model / pytorch_model, single-file or sharded, plus their index), NOT a bare + # *.safetensors: a repo that also publishes alternate variants would otherwise pull + # those unused shards too. These cover every file _resolve_fp8_scale_safetensors_files + # reads for the no-variant case, so nothing needed is skipped. #6749 + patterns = [ + "model.safetensors", + "pytorch_model.safetensors", + "model-*-of-*.safetensors", + "pytorch_model-*-of-*.safetensors", + "model.safetensors.index.json", + "pytorch_model.safetensors.index.json", + ] if subfolder: patterns = [f"{subfolder}/{pattern}" for pattern in patterns] return patterns From beb07a01c56ce9212b9789eb612ea6f12998fe0a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 7 Jul 2026 15:33:47 +0000 Subject: [PATCH 22/35] Preserve FP8 scale parameters and fetch index-named scale shards for PR #6749 --- tests/test_fp8_scale_inv_load.py | 80 ++++++++++++++++++++++++++++++++ unsloth/models/loader_utils.py | 61 ++++++++++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 77feaf5e8db3..3bd2a0f5ff73 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1227,3 +1227,83 @@ def test_fp8_probe_uses_absolute_optional_import_and_broad_nonfatal_guard(): return pytest.fail("_is_real_fp8_owner no longer protects the optional fp8 kernel import.") + + +def test_restored_scale_stays_a_parameter_when_placeholder_is_a_parameter(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) + # HF FP8Linear.weight_scale_inv / FbgemmFp8Linear.weight_scale are Parameters; the restore + # must keep them Parameters rather than delattr + register_buffer (which would silently + # demote the scale to a buffer and hide it from parameter inspection). + model.fp8.weight_scale_inv = torch.nn.Parameter( + torch.tensor([9.0], dtype = torch.float16), requires_grad = False + ) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.0], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert "weight_scale_inv" in model.fp8._parameters + assert "weight_scale_inv" not in model.fp8._buffers + assert isinstance(model.fp8.weight_scale_inv, torch.nn.Parameter) + assert torch.equal( + model.fp8.weight_scale_inv.detach(), + torch.tensor([1.0], dtype = torch.float16), + ) + + +def test_index_extra_patterns_cover_nonstandard_scale_shards(): + loader_utils = _load_loader_utils() + with tempfile.TemporaryDirectory() as model_dir: + index = { + "weight_map": { + "fp8.weight_scale_inv": "custom-shard-00001.safetensors", + "fp8.weight": "custom-shard-00001.safetensors", + "other.bias": "custom-shard-00002.safetensors", + } + } + with open( + os.path.join(model_dir, "model.safetensors.index.json"), + "w", + encoding = "utf-8", + ) as fh: + json.dump(index, fh) + extras = loader_utils._fp8_scale_index_extra_patterns(model_dir) + + # Only the shard that actually holds a scale tensor is fetched; weight-only and non-scale + # shards are left out. + assert extras == ["custom-shard-00001.safetensors"] + # The default snapshot globs do not match this non-standard shard name, so without the + # index-driven extra pass the scale shard would be under-fetched. + default_patterns = loader_utils._fp8_scale_snapshot_allow_patterns() + assert not any( + fnmatch.fnmatch("custom-shard-00001.safetensors", pattern) + for pattern in default_patterns + ) + + +def test_index_extra_patterns_prefix_subfolder(): + loader_utils = _load_loader_utils() + with tempfile.TemporaryDirectory() as model_dir: + subdir = os.path.join(model_dir, "sub") + os.makedirs(subdir) + index = {"weight_map": {"fp8.weight_scale_inv": "weights-00001.safetensors"}} + with open( + os.path.join(subdir, "model.safetensors.index.json"), + "w", + encoding = "utf-8", + ) as fh: + json.dump(index, fh) + extras = loader_utils._fp8_scale_index_extra_patterns(model_dir, subfolder = "sub") + + assert extras == ["sub/weights-00001.safetensors"] diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index c388ffc0908d..51b40a8ec2fe 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -563,6 +563,34 @@ def _fp8_scale_snapshot_allow_patterns( return patterns +def _fp8_scale_index_extra_patterns(model_dir, subfolder = None, variant = None): + """Exact safetensors shard names an FP8 scale index references that the default snapshot + globs above do not cover. `_resolve_fp8_scale_safetensors_files` follows the index + `weight_map` verbatim, so a repo that maps a scale tensor to a shard named outside + `model*`/`pytorch_model*` would never be downloaded and the restore would silently find + no scales even though the main load reads that shard by its index name. #6749""" + scan_dir = os.path.join(model_dir, subfolder) if subfolder else model_dir + if not os.path.isdir(scan_dir): + return [] + for index_name in _variant_index_names(variant): + index_path = os.path.join(scan_dir, index_name) + if not os.path.isfile(index_path): + continue + try: + with open(index_path, encoding = "utf-8") as fh: + weight_map = json.load(fh).get("weight_map", {}) + except Exception: + return [] + extras = [] + for key, filename in weight_map.items(): + if not any(key.endswith(suffix) for _, suffix in _FP8_SCALE_SUFFIXES): + continue + if isinstance(filename, str) and filename.endswith(".safetensors"): + extras.append(f"{subfolder}/{filename}" if subfolder else filename) + return sorted(set(extras)) + return [] + + def _find_fp8_scale_inv_tensors( model_dir, model_name, @@ -604,6 +632,29 @@ def _find_fp8_scale_inv_tensors( except Exception: return [] + # The default globs cover standard shard names; if the index instead maps scale tensors + # to shards named outside them, pull those exact shards too so the restore below can read + # them. Only the shards the first pass missed are fetched, so standard repos add nothing. + extra_patterns = _fp8_scale_index_extra_patterns(model_dir, subfolder, variant) + missing = [ + pattern + for pattern in extra_patterns + if not os.path.isfile(os.path.join(model_dir, pattern)) + ] + if missing: + try: + model_dir = snapshot_download( + model_name, + revision = revision, + token = token, + local_files_only = local_files_only, + cache_dir = cache_dir, + force_download = force_download, + allow_patterns = allow_patterns + missing, + ) + except Exception: + pass + if not os.path.isdir(model_dir): return [] @@ -778,6 +829,16 @@ def _restore_missing_fp8_weight_scale_inv( pass if attr_name in module._buffers: module._buffers[attr_name] = restored_scale + elif attr_name in module._parameters: + # HF FP8Linear.weight_scale_inv / FbgemmFp8Linear.weight_scale live in _parameters, + # so keep them Parameters here instead of delattr + register_buffer: demoting a scale + # to a buffer changes the module's parameter set and breaks tensor-parallel/optimizer/ + # PEFT parameter inspection that expects these FP8 scales to stay parameters. #6749 + existing = module._parameters[attr_name] + requires_grad = bool(getattr(existing, "requires_grad", False)) + module._parameters[attr_name] = torch.nn.Parameter( + restored_scale, requires_grad = requires_grad + ) else: if hasattr(module, attr_name): delattr(module, attr_name) From db79e37bfe03df82761c9935c4bb9ef3f9dcfc85 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:34:41 +0000 Subject: [PATCH 23/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_fp8_scale_inv_load.py | 3 +-- unsloth/models/loader_utils.py | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 3bd2a0f5ff73..35f1ffc3697a 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1287,8 +1287,7 @@ def test_index_extra_patterns_cover_nonstandard_scale_shards(): # index-driven extra pass the scale shard would be under-fetched. default_patterns = loader_utils._fp8_scale_snapshot_allow_patterns() assert not any( - fnmatch.fnmatch("custom-shard-00001.safetensors", pattern) - for pattern in default_patterns + fnmatch.fnmatch("custom-shard-00001.safetensors", pattern) for pattern in default_patterns ) diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 51b40a8ec2fe..d2f6cf699c27 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -563,7 +563,11 @@ def _fp8_scale_snapshot_allow_patterns( return patterns -def _fp8_scale_index_extra_patterns(model_dir, subfolder = None, variant = None): +def _fp8_scale_index_extra_patterns( + model_dir, + subfolder = None, + variant = None, +): """Exact safetensors shard names an FP8 scale index references that the default snapshot globs above do not cover. `_resolve_fp8_scale_safetensors_files` follows the index `weight_map` verbatim, so a repo that maps a scale tensor to a shard named outside From 554ae3acf382ff54cccf6debf12eed78e9a9897e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 7 Jul 2026 16:02:07 +0000 Subject: [PATCH 24/35] Restore transposed FP8 scales and preserve scale parameter metadata for PR #6749 --- tests/test_fp8_scale_inv_load.py | 79 ++++++++++++++++++++++++++++++++ unsloth/models/loader_utils.py | 19 ++++++-- 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 35f1ffc3697a..1e85da270720 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1306,3 +1306,82 @@ def test_index_extra_patterns_prefix_subfolder(): extras = loader_utils._fp8_scale_index_extra_patterns(model_dir, subfolder = "sub") assert extras == ["sub/weights-00001.safetensors"] + + +def test_restores_transposed_column_wise_scale(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + # weight is [out=2, in=4]; a transposed/column-wise scale has shape[0] == in == 4, which + # exceeds out but is a layout the FP8 kernel supports and must not be skipped. + model.fp8 = _Fp8Owner(weight = torch.randn(2, 4, dtype = torch.float16)) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.0, 2.0, 3.0, 4.0], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert tuple(model.fp8.weight_scale_inv.shape) == (4,) + + +def test_skips_scale_that_matches_neither_weight_dim(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(2, 4, dtype = torch.float16)) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + # 5 rows matches neither out(2) nor in(4): a genuinely oversized scale, still skipped. + {"fp8.weight_scale_inv": torch.arange(5, dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 0 + assert skipped == 1 + + +def test_parameter_scale_restore_preserves_metadata_and_identity(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) + placeholder = torch.nn.Parameter( + torch.tensor([2.0], dtype = torch.float16), requires_grad = False + ) + # Non-default block geometry the fp8 kernels read via getattr(weight_scale, "block_size", ...). + placeholder.block_size = [64, 64] + model.fp8.weight_scale_inv = placeholder + placeholder_id = id(model.fp8.weight_scale_inv) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.0], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert isinstance(model.fp8.weight_scale_inv, torch.nn.Parameter) + # Same Parameter object, so custom metadata survives the restore. + assert id(model.fp8.weight_scale_inv) == placeholder_id + assert getattr(model.fp8.weight_scale_inv, "block_size", None) == [64, 64] + assert torch.equal( + model.fp8.weight_scale_inv.detach(), + torch.tensor([1.0], dtype = torch.float16), + ) diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index d2f6cf699c27..f508c3b99cbd 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -808,6 +808,10 @@ def _restore_missing_fp8_weight_scale_inv( and scale_tensor.ndim > 0 and weight.ndim > 0 and scale_tensor.shape[0] > weight.shape[0] + # A transposed/column-wise scale has shape[0] == weight.shape[1] and is a layout the + # FP8 kernel handles (see unsloth/kernels/fp8.py transposed weight/scale branch), so + # only reject scales whose row count matches neither the weight rows nor columns. #6749 + and (weight.ndim < 2 or scale_tensor.shape[0] != weight.shape[1]) ): skipped += 1 continue @@ -839,10 +843,17 @@ def _restore_missing_fp8_weight_scale_inv( # to a buffer changes the module's parameter set and breaks tensor-parallel/optimizer/ # PEFT parameter inspection that expects these FP8 scales to stay parameters. #6749 existing = module._parameters[attr_name] - requires_grad = bool(getattr(existing, "requires_grad", False)) - module._parameters[attr_name] = torch.nn.Parameter( - restored_scale, requires_grad = requires_grad - ) + if isinstance(existing, torch.nn.Parameter): + # Update in place so any runtime metadata the kernels read off the scale (e.g. + # `block_size`, which fp8 dequant pulls via getattr) survives; building a fresh + # Parameter would drop those attributes and force default block geometry. #6749 + with torch.no_grad(): + existing.data = restored_scale + else: + module._parameters[attr_name] = torch.nn.Parameter( + restored_scale, + requires_grad = bool(getattr(existing, "requires_grad", False)), + ) else: if hasattr(module, attr_name): delattr(module, attr_name) From 88903aaeda7b3d20ea22a4605208f6c6e0b081d2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:03:09 +0000 Subject: [PATCH 25/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_fp8_scale_inv_load.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 1e85da270720..320f8442d52e 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1356,9 +1356,7 @@ def test_parameter_scale_restore_preserves_metadata_and_identity(): loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) - placeholder = torch.nn.Parameter( - torch.tensor([2.0], dtype = torch.float16), requires_grad = False - ) + placeholder = torch.nn.Parameter(torch.tensor([2.0], dtype = torch.float16), requires_grad = False) # Non-default block geometry the fp8 kernels read via getattr(weight_scale, "block_size", ...). placeholder.block_size = [64, 64] model.fp8.weight_scale_inv = placeholder From bc71f4124c45568ce6a5346df4e2d4a2ba0d00ce Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Jul 2026 01:46:52 +0000 Subject: [PATCH 26/35] Tighten comments for PR #6749 --- tests/test_fp8_scale_inv_load.py | 50 +++++++++++++++++++------------- unsloth/models/loader.py | 5 ++-- unsloth/models/loader_utils.py | 43 ++++++++++----------------- 3 files changed, 47 insertions(+), 51 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 320f8442d52e..d5fd78a67ec9 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1,3 +1,19 @@ +# Unsloth Zoo - Utilities for Unsloth +# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import ast import fnmatch import importlib.util @@ -355,8 +371,8 @@ def recording_safe_open(path, *args, **kwargs): def test_remote_scale_restore_constrains_snapshot_download_patterns(monkeypatch): - # A remote FP8 scale restore must not re-pull the whole repo after the model already - # loaded: constrain snapshot_download to the index + selected safetensors, never .bin. #6749 + # A remote scale restore must constrain snapshot_download to the index + selected + # safetensors, not re-pull the whole repo. #6749 loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _PackedFp8Owner() @@ -390,8 +406,8 @@ def fake_snapshot_download(repo_id, **kwargs): def test_no_variant_snapshot_patterns_exclude_alternate_variants(): - # The no-variant patterns must cover the default single-file/sharded/index names the scanner - # opens, but NOT alternate variant safetensors a repo may also publish. #6749 + # No-variant patterns cover default single-file/sharded/index names, not alternate + # variant safetensors a repo may also publish. #6749 import fnmatch loader_utils = _load_loader_utils() @@ -446,17 +462,15 @@ def fake_snapshot_download(repo_id, **kwargs): assert skipped == 0 allow_patterns = recorded["allow_patterns"] assert allow_patterns is not None - # Every pattern must be variant-scoped (mentions the variant) and none may be the bare - # non-variant safetensors/index globs, so the snapshot stays limited to the selected variant. + # Every pattern must be variant-scoped, none the bare non-variant globs, so the + # snapshot stays limited to the selected variant. assert all("fp8" in p for p in allow_patterns) assert not any(p in ("*.safetensors", "*.safetensors.index.json") for p in allow_patterns) def test_remote_scale_restore_snapshot_patterns_cover_sharded_variant_shards(monkeypatch): - # transformers names a sharded variant shard `model.-00001-of-00002.safetensors` - # (variant applied before the shard suffix). The scale-restore allow_patterns must cover that - # form, or snapshot_download pulls the variant index but none of its shards and no scales are - # restored - a silent regression for remote sharded variant FP8 checkpoints. #6749 + # allow_patterns must cover the sharded variant form + # `model.-00001-of-00002.safetensors`, else the index downloads but no shards. #6749 loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _PackedFp8Owner() @@ -501,9 +515,8 @@ def fake_snapshot_download(repo_id, **kwargs): def test_restored_floating_owner_scale_matches_weight_dtype(): - # A floating (fp16/bf16) FP8 owner with no scale placeholder has no reference dtype to match, - # so the restored buffer must mirror the weight dtype (like the placeholder path) rather than - # keep the checkpoint's fp32, which would disagree with the module dtype at matmul. #6749 + # A floating (fp16/bf16) FP8 owner with no scale placeholder must get a buffer mirroring + # the weight dtype, not the checkpoint fp32 that would mismatch at matmul. #6749 loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.bfloat16)) @@ -1061,10 +1074,8 @@ def test_fastmodel_peft_base_mapping_clears_fp8_after_remap(): def test_fastmodel_peft_preserves_adapter_name_for_peft_load(): - # The FP8 base remap must compare against a pre-remap base name, but that comparison must - # NOT reuse old_model_name: old_model_name still points at the adapter repo that - # PeftModel.from_pretrained loads, so clobbering it inside the is_peft block would make - # PEFT attach the base checkpoint instead of the adapter. #6749 + # The FP8 base remap must compare against a pre-remap base name without reusing + # old_model_name, which still points at the adapter repo PeftModel loads. #6749 tree = ast.parse(LOADER.read_text()) offending = [] @@ -1233,9 +1244,8 @@ def test_restored_scale_stays_a_parameter_when_placeholder_is_a_parameter(): loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) - # HF FP8Linear.weight_scale_inv / FbgemmFp8Linear.weight_scale are Parameters; the restore - # must keep them Parameters rather than delattr + register_buffer (which would silently - # demote the scale to a buffer and hide it from parameter inspection). + # HF FP8 scales are Parameters; the restore must keep them Parameters, not demote them + # to buffers hidden from parameter inspection. model.fp8.weight_scale_inv = torch.nn.Parameter( torch.tensor([9.0], dtype = torch.float16), requires_grad = False ) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 13c3f30e70f1..c14f4a2cbcdc 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -1576,9 +1576,8 @@ def _dispatch_diffusion(): # Check base model again for PEFT model_name = peft_config.base_model_name_or_path if not use_exact_model_name: - # Compare against a separate local, not old_model_name: old_model_name still - # holds the adapter repo that PeftModel.from_pretrained loads below, so reusing - # it here would attach the base checkpoint instead of the adapter. #6749 + # Use a separate local, not old_model_name, which still holds the adapter repo + # PeftModel.from_pretrained loads below; reusing it would attach the base. #6749 base_before_remap = model_name model_name = get_model_name( model_name, diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index f508c3b99cbd..a10d077aa31e 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -535,21 +535,16 @@ def _fp8_scale_snapshot_allow_patterns( if use_safetensors is False: return [] if variant: - # transformers applies the variant BEFORE the shard suffix, so a sharded variant shard - # is `model.-00001-of-00002.safetensors` (matched by `*.-*.safetensors`), - # not `model..safetensors` (only the single-file form). Cover both, else - # snapshot_download fetches the variant index but none of its shards and no scales load. #6749 + # Cover single-file and sharded variant forms; transformers applies the variant before + # the shard suffix, so a sharded shard is `model.-00001-of-00002.safetensors`. #6749 patterns = [ f"*.{variant}.safetensors", f"*.{variant}-*.safetensors", f"*.safetensors.index.{variant}.json", ] else: - # Restrict to the default (non-variant) transformers artifact names the scanner opens - # (model / pytorch_model, single-file or sharded, plus their index), NOT a bare - # *.safetensors: a repo that also publishes alternate variants would otherwise pull - # those unused shards too. These cover every file _resolve_fp8_scale_safetensors_files - # reads for the no-variant case, so nothing needed is skipped. #6749 + # Default (non-variant) model/pytorch_model artifacts plus index, not a bare + # *.safetensors that would also pull alternate variant shards. #6749 patterns = [ "model.safetensors", "pytorch_model.safetensors", @@ -636,9 +631,8 @@ def _find_fp8_scale_inv_tensors( except Exception: return [] - # The default globs cover standard shard names; if the index instead maps scale tensors - # to shards named outside them, pull those exact shards too so the restore below can read - # them. Only the shards the first pass missed are fetched, so standard repos add nothing. + # Fetch any scale shards the index maps to names outside the default globs; the first + # pass covers standard names, so standard repos add nothing here. extra_patterns = _fp8_scale_index_extra_patterns(model_dir, subfolder, variant) missing = [ pattern @@ -695,8 +689,7 @@ def _find_fp8_scale_inv_tensors( ] -# Projection tensors kept by weightless FP8 experts (FP8Experts / -# FbgemmFp8Llama4TextExperts) in place of a top-level `weight`. +# Projections weightless FP8 experts keep in place of a top-level `weight`. _FP8_PROJECTION_ATTRS = ("gate_up_proj", "gate_proj", "up_proj", "down_proj") @@ -808,9 +801,8 @@ def _restore_missing_fp8_weight_scale_inv( and scale_tensor.ndim > 0 and weight.ndim > 0 and scale_tensor.shape[0] > weight.shape[0] - # A transposed/column-wise scale has shape[0] == weight.shape[1] and is a layout the - # FP8 kernel handles (see unsloth/kernels/fp8.py transposed weight/scale branch), so - # only reject scales whose row count matches neither the weight rows nor columns. #6749 + # Keep transposed/column-wise scales (shape[0] == weight.shape[1]) the FP8 kernel + # handles; reject only scales matching neither weight rows nor columns. #6749 and (weight.ndim < 2 or scale_tensor.shape[0] != weight.shape[1]) ): skipped += 1 @@ -827,10 +819,8 @@ def _restore_missing_fp8_weight_scale_inv( and weight.is_floating_point() and not _has_float8_dtype(weight.dtype) ): - # Floating (fp16/bf16) FP8 owner whose scale placeholder was dropped: no reference to - # match, so mirror the weight dtype like the placeholder path does instead of leaving - # the buffer in the checkpoint's fp32. Packed (int8) and real float8 weights keep the - # checkpoint dtype. #6749 + # Floating (fp16/bf16) FP8 owner with no reference scale: mirror the weight dtype + # instead of the checkpoint fp32. Packed/int8 and real float8 keep checkpoint dtype. #6749 try: restored_scale = restored_scale.to(dtype = weight.dtype) except Exception: @@ -838,15 +828,12 @@ def _restore_missing_fp8_weight_scale_inv( if attr_name in module._buffers: module._buffers[attr_name] = restored_scale elif attr_name in module._parameters: - # HF FP8Linear.weight_scale_inv / FbgemmFp8Linear.weight_scale live in _parameters, - # so keep them Parameters here instead of delattr + register_buffer: demoting a scale - # to a buffer changes the module's parameter set and breaks tensor-parallel/optimizer/ - # PEFT parameter inspection that expects these FP8 scales to stay parameters. #6749 + # HF FP8 scales live in _parameters; keep them Parameters rather than demoting to a + # buffer, which would break tensor-parallel/optimizer/PEFT parameter inspection. #6749 existing = module._parameters[attr_name] if isinstance(existing, torch.nn.Parameter): - # Update in place so any runtime metadata the kernels read off the scale (e.g. - # `block_size`, which fp8 dequant pulls via getattr) survives; building a fresh - # Parameter would drop those attributes and force default block geometry. #6749 + # Update in place so runtime metadata (e.g. `block_size` fp8 dequant reads via + # getattr) survives; a fresh Parameter would drop it and force default geometry. #6749 with torch.no_grad(): existing.data = restored_scale else: From 7650ed766f8fab5c25a06aa3ec663ad0a89b1415 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Jul 2026 02:44:05 +0000 Subject: [PATCH 27/35] Narrow variant FP8 globs, restore meta-placeholder and static activation scales for PR #6749 --- tests/test_fp8_scale_inv_load.py | 64 ++++++++++++++++++++++++++++++++ unsloth/models/loader_utils.py | 30 ++++++++++++--- 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index d5fd78a67ec9..4b6842607e80 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1393,3 +1393,67 @@ def test_parameter_scale_restore_preserves_metadata_and_identity(): model.fp8.weight_scale_inv.detach(), torch.tensor([1.0], dtype = torch.float16), ) + + +def test_variant_snapshot_patterns_exclude_alternate_component_safetensors(): + loader_utils = _load_loader_utils() + patterns = loader_utils._fp8_scale_snapshot_allow_patterns(variant = "fp8") + # Default variant model shards and index are covered. + assert any(fnmatch.fnmatch("model.fp8.safetensors", p) for p in patterns) + assert any(fnmatch.fnmatch("model.fp8-00001-of-00002.safetensors", p) for p in patterns) + assert any(fnmatch.fnmatch("model.safetensors.index.fp8.json", p) for p in patterns) + # A stray alternate-component variant safetensors must NOT be pulled by the broad glob. + assert not any(fnmatch.fnmatch("adapter.fp8.safetensors", p) for p in patterns) + assert not any(fnmatch.fnmatch("vision_tower.fp8.safetensors", p) for p in patterns) + + +def test_restores_scale_when_placeholder_is_meta_but_weight_is_live(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) + # Placeholder dropped to meta (device-map/low-memory) while the weight is materialized. + model.fp8.register_buffer( + "weight_scale_inv", torch.empty(1, dtype = torch.float16, device = "meta") + ) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.5], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert model.fp8.weight_scale_inv.device.type == "cpu" + assert torch.equal( + model.fp8.weight_scale_inv, torch.tensor([1.5], dtype = torch.float16) + ) + + +def test_restores_static_activation_scale(): + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.activation_scale": torch.tensor([0.25], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert hasattr(model.fp8, "activation_scale") + assert torch.equal( + model.fp8.activation_scale, torch.tensor([0.25], dtype = torch.float16) + ) diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index a10d077aa31e..a5720e14f58b 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -442,6 +442,13 @@ def _model_has_real_fp8_modules(model): ("gate_up_proj_scale_inv", ".gate_up_proj_scale_inv"), ("down_proj_scale", ".down_proj_scale"), ("down_proj_scale_inv", ".down_proj_scale_inv"), + # Static-activation FP8 checkpoints carry activation scales the expert forward reads when + # activation_scheme == "static"; restore them too so a dropped placeholder is repaired. #6749 + ("activation_scale", ".activation_scale"), + ("gate_proj_activation_scale", ".gate_proj_activation_scale"), + ("up_proj_activation_scale", ".up_proj_activation_scale"), + ("gate_up_proj_activation_scale", ".gate_up_proj_activation_scale"), + ("down_proj_activation_scale", ".down_proj_activation_scale"), ) @@ -535,12 +542,16 @@ def _fp8_scale_snapshot_allow_patterns( if use_safetensors is False: return [] if variant: - # Cover single-file and sharded variant forms; transformers applies the variant before - # the shard suffix, so a sharded shard is `model.-00001-of-00002.safetensors`. #6749 + # Default model/pytorch_model variant artifacts plus index, not bare *.{variant} + # globs that would also pull alternate component safetensors; custom shard names come + # from the index-extra pass. Transformers applies the variant before the shard suffix. #6749 patterns = [ - f"*.{variant}.safetensors", - f"*.{variant}-*.safetensors", - f"*.safetensors.index.{variant}.json", + f"model.{variant}.safetensors", + f"pytorch_model.{variant}.safetensors", + f"model.{variant}-*-of-*.safetensors", + f"pytorch_model.{variant}-*-of-*.safetensors", + f"model.safetensors.index.{variant}.json", + f"pytorch_model.safetensors.index.{variant}.json", ] else: # Default (non-variant) model/pytorch_model artifacts plus index, not a bare @@ -773,7 +784,14 @@ def _restore_missing_fp8_weight_scale_inv( ): skipped += 1 continue - if isinstance(reference, torch.Tensor): + # Prefer a materialized device: a meta scale placeholder can sit next to an already + # loaded weight (device-map/low-memory), so fall back to the live weight/projection + # device rather than skipping on the placeholder's meta device. #6749 + if isinstance(reference, torch.Tensor) and reference.device.type != "meta": + target_device = reference.device + elif isinstance(weight, torch.Tensor) and weight.device.type != "meta": + target_device = weight.device + elif isinstance(reference, torch.Tensor): target_device = reference.device elif isinstance(weight, torch.Tensor): target_device = weight.device From b3165b3208af652b19fcdb183c788fb6b522be58 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 02:45:07 +0000 Subject: [PATCH 28/35] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_fp8_scale_inv_load.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 4b6842607e80..715777a913c3 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1430,9 +1430,7 @@ def test_restores_scale_when_placeholder_is_meta_but_weight_is_live(): assert restored == 1 assert skipped == 0 assert model.fp8.weight_scale_inv.device.type == "cpu" - assert torch.equal( - model.fp8.weight_scale_inv, torch.tensor([1.5], dtype = torch.float16) - ) + assert torch.equal(model.fp8.weight_scale_inv, torch.tensor([1.5], dtype = torch.float16)) def test_restores_static_activation_scale(): @@ -1454,6 +1452,4 @@ def test_restores_static_activation_scale(): assert restored == 1 assert skipped == 0 assert hasattr(model.fp8, "activation_scale") - assert torch.equal( - model.fp8.activation_scale, torch.tensor([0.25], dtype = torch.float16) - ) + assert torch.equal(model.fp8.activation_scale, torch.tensor([0.25], dtype = torch.float16)) From 0d6c49c98cc24280c81765b014f47f87cce0777f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Jul 2026 04:26:45 +0000 Subject: [PATCH 29/35] Guard FP8 cache regeneration, meta scale parameters, and index-only shard fetch for PR #6749 --- tests/test_fp8_scale_inv_load.py | 89 ++++++++++++++++++++++++++++---- unsloth/models/loader_utils.py | 52 ++++++++++--------- 2 files changed, 107 insertions(+), 34 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 715777a913c3..586fa4e735b8 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -405,9 +405,10 @@ def fake_snapshot_download(repo_id, **kwargs): assert all("bin" not in pattern for pattern in allow_patterns) -def test_no_variant_snapshot_patterns_exclude_alternate_variants(): - # No-variant patterns cover default single-file/sharded/index names, not alternate - # variant safetensors a repo may also publish. #6749 +def test_no_variant_snapshot_patterns_are_index_and_single_file_only(): + # First-pass patterns cover only the default single-file names and index; shards are pulled + # by exact name in the index-extra pass, so shard globs and alternate variants are excluded + # and a large sharded checkpoint is not fully re-downloaded. #6749 import fnmatch loader_utils = _load_loader_utils() @@ -418,12 +419,14 @@ def matched(name): for name in ( "model.safetensors", - "model-00001-of-00002.safetensors", - "pytorch_model-00001-of-00002.safetensors", + "pytorch_model.safetensors", "model.safetensors.index.json", + "pytorch_model.safetensors.index.json", ): assert matched(name), name for name in ( + "model-00001-of-00002.safetensors", + "pytorch_model-00001-of-00002.safetensors", "model.fp8.safetensors", "model.fp8-00001-of-00002.safetensors", "pytorch_model.bin", @@ -1395,14 +1398,15 @@ def test_parameter_scale_restore_preserves_metadata_and_identity(): ) -def test_variant_snapshot_patterns_exclude_alternate_component_safetensors(): +def test_variant_snapshot_patterns_are_index_and_single_file_only(): loader_utils = _load_loader_utils() patterns = loader_utils._fp8_scale_snapshot_allow_patterns(variant = "fp8") - # Default variant model shards and index are covered. + # First pass covers the single-file variant name and the variant index only. assert any(fnmatch.fnmatch("model.fp8.safetensors", p) for p in patterns) - assert any(fnmatch.fnmatch("model.fp8-00001-of-00002.safetensors", p) for p in patterns) assert any(fnmatch.fnmatch("model.safetensors.index.fp8.json", p) for p in patterns) - # A stray alternate-component variant safetensors must NOT be pulled by the broad glob. + # Shards come from the index-extra pass, so the shard glob is not in the first pass, and a + # stray alternate-component variant safetensors is never pulled. + assert not any(fnmatch.fnmatch("model.fp8-00001-of-00002.safetensors", p) for p in patterns) assert not any(fnmatch.fnmatch("adapter.fp8.safetensors", p) for p in patterns) assert not any(fnmatch.fnmatch("vision_tower.fp8.safetensors", p) for p in patterns) @@ -1453,3 +1457,70 @@ def test_restores_static_activation_scale(): assert skipped == 0 assert hasattr(model.fp8, "activation_scale") assert torch.equal(model.fp8.activation_scale, torch.tensor([0.25], dtype = torch.float16)) + + +def test_restores_scale_when_placeholder_is_meta_parameter(): + # A meta Parameter placeholder cannot take `.data = ` (torch rejects + # meta -> concrete set_data); the restore must replace it instead of crashing. #6749 + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) + model.fp8.weight_scale_inv = torch.nn.Parameter( + torch.empty(1, dtype = torch.float16, device = "meta"), requires_grad = False + ) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.75], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert isinstance(model.fp8.weight_scale_inv, torch.nn.Parameter) + assert model.fp8.weight_scale_inv.device.type == "cpu" + assert torch.equal( + model.fp8.weight_scale_inv.detach(), torch.tensor([1.75], dtype = torch.float16) + ) + + +def test_offline_quantize_preserves_bin_cache_on_failed_regeneration(monkeypatch, tmp_path): + # A failed re-quantization (offline / gated / removed source) must not delete a usable + # bin-only cache before the safetensors save has succeeded. #6749 + loader_utils = _load_loader_utils() + import transformers + + monkeypatch.setattr(loader_utils.tempfile, "gettempdir", lambda: str(tmp_path)) + cache_dir = tmp_path / "mymodel-fp8-e4m3" + cache_dir.mkdir() + (cache_dir / "pytorch_model.bin").write_text("old-cache") + + class _Cfg: + architectures = [] + + def _raise(*args, **kwargs): + raise RuntimeError("model is gated/offline") + + def _make(from_pretrained): + return type("_Auto", (), {"from_pretrained": staticmethod(from_pretrained)}) + + monkeypatch.setattr(transformers, "AutoConfig", _make(lambda *a, **k: _Cfg())) + monkeypatch.setattr(transformers, "AutoModelForCausalLM", _make(_raise)) + monkeypatch.setattr(transformers, "AutoModelForImageTextToText", _make(_raise)) + monkeypatch.setattr(transformers, "AutoTokenizer", _make(lambda *a, **k: None)) + monkeypatch.setattr(transformers, "AutoProcessor", _make(lambda *a, **k: None)) + monkeypatch.setattr(transformers, "TorchAoConfig", lambda *a, **k: object()) + monkeypatch.setattr(loader_utils, "_get_torchao_fp8_config", lambda mode: {}) + + with pytest.raises(RuntimeError): + loader_utils._offline_quantize_to_fp8("org/mymodel", "e4m3") + + # The loadable bin cache survives, and no partial staging dir is left behind. + assert cache_dir.is_dir() + assert (cache_dir / "pytorch_model.bin").read_text() == "old-cache" + assert not (tmp_path / "mymodel-fp8-e4m3.fp8tmp").exists() diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index a5720e14f58b..58c0be0aec90 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -329,12 +329,12 @@ def _offline_quantize_to_fp8( new_model_name = os.path.join(temp_dir, cache_name) print(f"Unsloth: Quantizing '{model_name}' to fp8, using model_name='{new_model_name}' instead") - if os.path.isdir(new_model_name) and not any( + has_safetensors_cache = os.path.isdir(new_model_name) and any( filename.endswith(".safetensors") for filename in os.listdir(new_model_name) - ): - shutil.rmtree(new_model_name) + ) + needs_refresh = os.path.isdir(new_model_name) and not has_safetensors_cache - if not os.path.isdir(new_model_name): + if not os.path.isdir(new_model_name) or needs_refresh: from ._utils import _apply_text_only_key_mapping qconfig = _get_torchao_fp8_config(fp8_mode) @@ -345,18 +345,27 @@ def _offline_quantize_to_fp8( config = text_config auto_model = AutoModelForImageTextToText if is_vlm else AutoModelForCausalLM auto_processor = AutoProcessor if is_vlm else AutoTokenizer + # Regenerate into a staging dir and only replace an existing (bin-only) cache once the + # safetensors save has succeeded, so a failed re-quantization (offline / gated / removed + # source model) leaves the previous loadable cache intact rather than deleting it. #6749 + staging_dir = new_model_name + ".fp8tmp" + if os.path.isdir(staging_dir): + shutil.rmtree(staging_dir) model = auto_model.from_pretrained( model_name, config = config, **load_kwargs, ) tokenizer = auto_processor.from_pretrained(model_name) - model.save_pretrained(new_model_name, safe_serialization = True) + model.save_pretrained(staging_dir, safe_serialization = True) del model for _ in range(2): torch.cuda.empty_cache() gc.collect() - tokenizer.save_pretrained(new_model_name) + tokenizer.save_pretrained(staging_dir) + if os.path.isdir(new_model_name): + shutil.rmtree(new_model_name) + os.replace(staging_dir, new_model_name) return new_model_name @@ -535,32 +544,24 @@ def _fp8_scale_snapshot_allow_patterns( variant = None, use_safetensors = None, ): - """Glob patterns limiting the scale-restore snapshot to the same safetensors artifact - from_pretrained selected. Without this, snapshot_download would pull the FULL repo - (.bin shards, alternate variants, large extras) after the model already loaded, which - exhausts disk/network and defeats the safetensors-only selection. #6749""" + """First-pass patterns for the scale-restore snapshot: the index plus the single-file + default model/pytorch_model safetensors only, never the shard globs. Sharded scale shards + are then fetched by exact name from the index in a second pass (_fp8_scale_index_extra_patterns), + so a large sharded checkpoint is not fully re-downloaded and alternate variants/extras are + never pulled. #6749""" if use_safetensors is False: return [] if variant: - # Default model/pytorch_model variant artifacts plus index, not bare *.{variant} - # globs that would also pull alternate component safetensors; custom shard names come - # from the index-extra pass. Transformers applies the variant before the shard suffix. #6749 patterns = [ f"model.{variant}.safetensors", f"pytorch_model.{variant}.safetensors", - f"model.{variant}-*-of-*.safetensors", - f"pytorch_model.{variant}-*-of-*.safetensors", f"model.safetensors.index.{variant}.json", f"pytorch_model.safetensors.index.{variant}.json", ] else: - # Default (non-variant) model/pytorch_model artifacts plus index, not a bare - # *.safetensors that would also pull alternate variant shards. #6749 patterns = [ "model.safetensors", "pytorch_model.safetensors", - "model-*-of-*.safetensors", - "pytorch_model-*-of-*.safetensors", "model.safetensors.index.json", "pytorch_model.safetensors.index.json", ] @@ -574,11 +575,10 @@ def _fp8_scale_index_extra_patterns( subfolder = None, variant = None, ): - """Exact safetensors shard names an FP8 scale index references that the default snapshot - globs above do not cover. `_resolve_fp8_scale_safetensors_files` follows the index - `weight_map` verbatim, so a repo that maps a scale tensor to a shard named outside - `model*`/`pytorch_model*` would never be downloaded and the restore would silently find - no scales even though the main load reads that shard by its index name. #6749""" + """Exact safetensors shard names the FP8 scale index maps scale tensors to. The first + snapshot pass fetches only the index, so this returns just the scale-bearing shards + (`_resolve_fp8_scale_safetensors_files` follows the index `weight_map` verbatim); the + second pass downloads only these, never every model shard or an alternate variant. #6749""" scan_dir = os.path.join(model_dir, subfolder) if subfolder else model_dir if not os.path.isdir(scan_dir): return [] @@ -849,12 +849,14 @@ def _restore_missing_fp8_weight_scale_inv( # HF FP8 scales live in _parameters; keep them Parameters rather than demoting to a # buffer, which would break tensor-parallel/optimizer/PEFT parameter inspection. #6749 existing = module._parameters[attr_name] - if isinstance(existing, torch.nn.Parameter): + if isinstance(existing, torch.nn.Parameter) and existing.device.type != "meta": # Update in place so runtime metadata (e.g. `block_size` fp8 dequant reads via # getattr) survives; a fresh Parameter would drop it and force default geometry. #6749 with torch.no_grad(): existing.data = restored_scale else: + # A meta placeholder Parameter cannot take `.data = ` (torch rejects + # meta -> concrete set_data), so replace it to materialize the scale. #6749 module._parameters[attr_name] = torch.nn.Parameter( restored_scale, requires_grad = bool(getattr(existing, "requires_grad", False)), From c1eed80f3ab585ac8b9db8e7c5720fbbcbfc6b29 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Jul 2026 05:21:44 +0000 Subject: [PATCH 30/35] Prefer projection device, preserve meta scale metadata, bin-cache fallback, force safetensors for FP8 siblings in PR #6749 --- tests/test_fp8_scale_inv_load.py | 117 +++++++++++++++++++++++++++++-- unsloth/models/loader.py | 8 +++ unsloth/models/loader_utils.py | 52 +++++++++----- 3 files changed, 156 insertions(+), 21 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 586fa4e735b8..dd0a58b0b505 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1200,7 +1200,9 @@ def test_offline_fp8_cache_forces_safetensors_loads(): ): hit_lines.add(child.lineno) - assert len(hit_lines) == 2 + # Two loader paths, each forcing safetensors in both the offline-quantize branch and the + # mapped pre-quantized FP8 sibling branch: 4 assignments total. #6749 + assert len(hit_lines) == 4 def test_fp8_probe_skips_missing_optional_kernel_module(): @@ -1517,10 +1519,117 @@ def _make(from_pretrained): monkeypatch.setattr(transformers, "TorchAoConfig", lambda *a, **k: object()) monkeypatch.setattr(loader_utils, "_get_torchao_fp8_config", lambda mode: {}) - with pytest.raises(RuntimeError): - loader_utils._offline_quantize_to_fp8("org/mymodel", "e4m3") + # A preserved bin-only cache is returned as a fallback rather than propagating the error. + result = loader_utils._offline_quantize_to_fp8("org/mymodel", "e4m3") - # The loadable bin cache survives, and no partial staging dir is left behind. + assert result == str(cache_dir) assert cache_dir.is_dir() assert (cache_dir / "pytorch_model.bin").read_text() == "old-cache" assert not (tmp_path / "mymodel-fp8-e4m3.fp8tmp").exists() + + +def test_offline_quantize_raises_when_no_cache_to_fall_back_to(monkeypatch, tmp_path): + # With no prior cache to fall back to, a failed regeneration must still surface the error. + loader_utils = _load_loader_utils() + import transformers + + monkeypatch.setattr(loader_utils.tempfile, "gettempdir", lambda: str(tmp_path)) + + class _Cfg: + architectures = [] + + def _raise(*args, **kwargs): + raise RuntimeError("model is gated/offline") + + def _make(from_pretrained): + return type("_Auto", (), {"from_pretrained": staticmethod(from_pretrained)}) + + monkeypatch.setattr(transformers, "AutoConfig", _make(lambda *a, **k: _Cfg())) + monkeypatch.setattr(transformers, "AutoModelForCausalLM", _make(_raise)) + monkeypatch.setattr(transformers, "AutoModelForImageTextToText", _make(_raise)) + monkeypatch.setattr(transformers, "AutoTokenizer", _make(lambda *a, **k: None)) + monkeypatch.setattr(transformers, "AutoProcessor", _make(lambda *a, **k: None)) + monkeypatch.setattr(transformers, "TorchAoConfig", lambda *a, **k: object()) + monkeypatch.setattr(loader_utils, "_get_torchao_fp8_config", lambda mode: {}) + + with pytest.raises(RuntimeError): + loader_utils._offline_quantize_to_fp8("org/mymodel", "e4m3") + assert not (tmp_path / "mymodel-fp8-e4m3").exists() + assert not (tmp_path / "mymodel-fp8-e4m3.fp8tmp").exists() + + +class _ProjFp8Owner(torch.nn.Module): + def __init__(self, gate_up_proj): + super().__init__() + self.gate_up_proj = gate_up_proj + self.quant_method = "fp8" + + +def test_restores_projection_scale_when_placeholder_is_meta(): + # A projection-only FP8 expert (materialized gate_up_proj, no weight) whose scale placeholder + # is still meta must restore onto the live projection device, not skip on the meta ref. #6749 + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.expert = _ProjFp8Owner(torch.randn(4, 4, dtype = torch.float16)) + model.expert.register_buffer( + "gate_up_proj_scale_inv", torch.empty(1, dtype = torch.float16, device = "meta") + ) + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"expert.gate_up_proj_scale_inv": torch.tensor([1.5], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert model.expert.gate_up_proj_scale_inv.device.type == "cpu" + + +def test_meta_parameter_replacement_preserves_metadata(): + # Replacing a meta Parameter placeholder must carry over custom attributes (e.g. block_size + # the fp8 kernels read) rather than dropping them. #6749 + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) + placeholder = torch.nn.Parameter( + torch.empty(1, dtype = torch.float16, device = "meta"), requires_grad = False + ) + placeholder.block_size = [64, 64] + model.fp8.weight_scale_inv = placeholder + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.0], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert isinstance(model.fp8.weight_scale_inv, torch.nn.Parameter) + assert model.fp8.weight_scale_inv.device.type == "cpu" + assert getattr(model.fp8.weight_scale_inv, "block_size", None) == [64, 64] + + +def test_fp8_sibling_branch_forces_safetensors(): + import re + + src = LOADER.read_text() + # Both pre-quantized FP8 sibling branches must force safetensors right after clearing + # load_in_fp8, so a caller-supplied use_safetensors=False cannot make from_pretrained look + # for absent .bin weights in a safetensors-only FP8 repo. #6749 + pattern = re.compile( + r"restore_fp8_scales = True\s+load_in_fp8 = False\s+#[^\n]*\n(?:\s+#[^\n]*\n)*" + r"\s+kwargs\[.use_safetensors.\] = True" + ) + assert len(pattern.findall(src)) == 2 diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index c14f4a2cbcdc..ae89ac4d0fb9 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -514,6 +514,10 @@ def from_pretrained( if load_in_fp8 != False and new_model_name != old_model_name: restore_fp8_scales = True load_in_fp8 = False + # Pre-quantized FP8 siblings are safetensors-only, so force safetensors like + # the offline path above; otherwise a caller-supplied use_safetensors=False + # makes from_pretrained look for absent .bin weights and fail. #6749 + kwargs["use_safetensors"] = True # Check if pre-quantized models are allowed # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64) @@ -1214,6 +1218,10 @@ def from_pretrained( if load_in_fp8 != False and new_model_name != old_model_name: restore_fp8_scales = True load_in_fp8 = False + # Pre-quantized FP8 siblings are safetensors-only, so force safetensors like + # the offline path above; otherwise a caller-supplied use_safetensors=False + # makes from_pretrained look for absent .bin weights and fail. #6749 + kwargs["use_safetensors"] = True # Check if pre-quantized models are allowed # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64) diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 58c0be0aec90..4b574ede60e1 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -351,18 +351,28 @@ def _offline_quantize_to_fp8( staging_dir = new_model_name + ".fp8tmp" if os.path.isdir(staging_dir): shutil.rmtree(staging_dir) - model = auto_model.from_pretrained( - model_name, - config = config, - **load_kwargs, - ) - tokenizer = auto_processor.from_pretrained(model_name) - model.save_pretrained(staging_dir, safe_serialization = True) - del model - for _ in range(2): - torch.cuda.empty_cache() - gc.collect() - tokenizer.save_pretrained(staging_dir) + try: + model = auto_model.from_pretrained( + model_name, + config = config, + **load_kwargs, + ) + tokenizer = auto_processor.from_pretrained(model_name) + model.save_pretrained(staging_dir, safe_serialization = True) + del model + for _ in range(2): + torch.cuda.empty_cache() + gc.collect() + tokenizer.save_pretrained(staging_dir) + except Exception: + # Regeneration failed (offline / gated / removed source). If a previous bin-only + # cache is still on disk, fall back to it so an already-usable cache keeps loading + # after an upgrade instead of hard-failing. #6749 + if os.path.isdir(staging_dir): + shutil.rmtree(staging_dir, ignore_errors = True) + if needs_refresh and os.path.isdir(new_model_name): + return new_model_name + raise if os.path.isdir(new_model_name): shutil.rmtree(new_model_name) os.replace(staging_dir, new_model_name) @@ -785,18 +795,21 @@ def _restore_missing_fp8_weight_scale_inv( skipped += 1 continue # Prefer a materialized device: a meta scale placeholder can sit next to an already - # loaded weight (device-map/low-memory), so fall back to the live weight/projection - # device rather than skipping on the placeholder's meta device. #6749 + # loaded weight/projection (device-map/low-memory), so a live device is tried before any + # meta placeholder. A projection-only expert has no weight, so its live projection device + # is preferred over a meta reference too, else its scale would be skipped. #6749 + projection_device = _fp8_projection_device(module, attr_name) if isinstance(reference, torch.Tensor) and reference.device.type != "meta": target_device = reference.device elif isinstance(weight, torch.Tensor) and weight.device.type != "meta": target_device = weight.device + elif projection_device is not None and projection_device.type != "meta": + target_device = projection_device elif isinstance(reference, torch.Tensor): target_device = reference.device elif isinstance(weight, torch.Tensor): target_device = weight.device else: - projection_device = _fp8_projection_device(module, attr_name) target_device = ( projection_device if projection_device is not None else scale_tensor.device ) @@ -856,11 +869,16 @@ def _restore_missing_fp8_weight_scale_inv( existing.data = restored_scale else: # A meta placeholder Parameter cannot take `.data = ` (torch rejects - # meta -> concrete set_data), so replace it to materialize the scale. #6749 - module._parameters[attr_name] = torch.nn.Parameter( + # meta -> concrete set_data), so replace it to materialize the scale, copying the + # existing parameter's custom attributes (e.g. `block_size` the fp8 kernels read) + # so non-default block geometry is not lost. #6749 + new_param = torch.nn.Parameter( restored_scale, requires_grad = bool(getattr(existing, "requires_grad", False)), ) + for meta_key, meta_val in vars(existing).items(): + setattr(new_param, meta_key, meta_val) + module._parameters[attr_name] = new_param else: if hasattr(module, attr_name): delattr(module, attr_name) From d5b499f11b7353ffff012286e243b6c1a8b4f410 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Jul 2026 05:49:35 +0000 Subject: [PATCH 31/35] Gate FP8 bin-cache fallback safetensors forcing and force safetensors for PEFT FP8 base remaps in PR #6749 --- tests/test_fp8_scale_inv_load.py | 26 ++++++++++++++++++++++++++ unsloth/models/loader.py | 15 +++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index dd0a58b0b505..5a657f3cb4d7 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1633,3 +1633,29 @@ def test_fp8_sibling_branch_forces_safetensors(): r"\s+kwargs\[.use_safetensors.\] = True" ) assert len(pattern.findall(src)) == 2 + + +def test_offline_fp8_bin_cache_fallback_not_forced_to_safetensors(): + import re + + src = LOADER.read_text() + # The offline quantize helper may fall back to a preserved bin-only cache; forcing safetensors + # afterwards must be guarded by a check that the returned dir actually has safetensors, else + # the bin fallback is unloadable. Both loader paths must do this. #6749 + pattern = re.compile( + r"_offline_quantize_to_fp8\([^\n]*\)\s+restore_fp8_scales = True\s+#[^\n]*\n(?:\s+#[^\n]*\n)*" + r'\s+if any\(name\.endswith\("\.safetensors"\)[^\n]*\n\s+kwargs\[.use_safetensors.\] = True' + ) + assert len(pattern.findall(src)) == 2 + + +def test_peft_fp8_base_remap_forces_safetensors(): + import re + + src = LOADER.read_text() + # The PEFT base remap to a pre-quantized FP8 sibling must also force safetensors. #6749 + pattern = re.compile( + r"model_name != base_before_remap:\s+load_in_fp8 = False\s+#[^\n]*\n(?:\s+#[^\n]*\n)*" + r"\s+kwargs\[.use_safetensors.\] = True" + ) + assert len(pattern.findall(src)) >= 1 diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index ae89ac4d0fb9..d4d422c12fc0 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -504,8 +504,12 @@ def from_pretrained( load_in_16bit, ) model_name = _offline_quantize_to_fp8(model_name, fp8_mode, text_only = text_only) - kwargs["use_safetensors"] = True restore_fp8_scales = True + # The helper may fall back to a preserved bin-only cache when regeneration fails; + # only force safetensors when it actually produced one, else from_pretrained would + # look for absent safetensors in that bin cache. #6749 + if any(name.endswith(".safetensors") for name in os.listdir(model_name)): + kwargs["use_safetensors"] = True else: assert new_model_name is not None model_name = new_model_name @@ -1208,8 +1212,12 @@ def from_pretrained( load_in_16bit, ) model_name = _offline_quantize_to_fp8(model_name, fp8_mode, text_only = text_only) - kwargs["use_safetensors"] = True restore_fp8_scales = True + # The helper may fall back to a preserved bin-only cache when regeneration fails; + # only force safetensors when it actually produced one, else from_pretrained would + # look for absent safetensors in that bin cache. #6749 + if any(name.endswith(".safetensors") for name in os.listdir(model_name)): + kwargs["use_safetensors"] = True else: assert new_model_name is not None model_name = new_model_name @@ -1594,6 +1602,9 @@ def _dispatch_diffusion(): ) if load_in_fp8 != False and model_name != base_before_remap: load_in_fp8 = False + # Pre-quantized FP8 base siblings are safetensors-only; force safetensors so a + # caller-supplied use_safetensors=False does not fail on absent .bin weights. #6749 + kwargs["use_safetensors"] = True # Check if pre-quantized models are allowed # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64) if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith( From 9a57062b57b4223db2893e0a06c4cfa218a5c1c3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Jul 2026 06:24:27 +0000 Subject: [PATCH 32/35] Tighten comments for PR #6749 --- tests/test_fp8_scale_inv_load.py | 60 ++++++++++---------------------- unsloth/models/loader.py | 22 ++++-------- unsloth/models/loader_utils.py | 40 ++++++++------------- 3 files changed, 39 insertions(+), 83 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 5a657f3cb4d7..be46c6c5ac75 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -371,8 +371,7 @@ def recording_safe_open(path, *args, **kwargs): def test_remote_scale_restore_constrains_snapshot_download_patterns(monkeypatch): - # A remote scale restore must constrain snapshot_download to the index + selected - # safetensors, not re-pull the whole repo. #6749 + # A remote scale restore must limit snapshot_download to the index + selected safetensors, not the whole repo. #6749 loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _PackedFp8Owner() @@ -406,9 +405,7 @@ def fake_snapshot_download(repo_id, **kwargs): def test_no_variant_snapshot_patterns_are_index_and_single_file_only(): - # First-pass patterns cover only the default single-file names and index; shards are pulled - # by exact name in the index-extra pass, so shard globs and alternate variants are excluded - # and a large sharded checkpoint is not fully re-downloaded. #6749 + # First pass covers only default single-file names and index; shards come by exact name later, so a large checkpoint is not fully re-downloaded. #6749 import fnmatch loader_utils = _load_loader_utils() @@ -465,15 +462,13 @@ def fake_snapshot_download(repo_id, **kwargs): assert skipped == 0 allow_patterns = recorded["allow_patterns"] assert allow_patterns is not None - # Every pattern must be variant-scoped, none the bare non-variant globs, so the - # snapshot stays limited to the selected variant. + # Every pattern must be variant-scoped (no bare globs) so the snapshot stays limited to the selected variant. assert all("fp8" in p for p in allow_patterns) assert not any(p in ("*.safetensors", "*.safetensors.index.json") for p in allow_patterns) def test_remote_scale_restore_snapshot_patterns_cover_sharded_variant_shards(monkeypatch): - # allow_patterns must cover the sharded variant form - # `model.-00001-of-00002.safetensors`, else the index downloads but no shards. #6749 + # allow_patterns must cover the sharded variant form, else the index downloads but no shards. #6749 loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _PackedFp8Owner() @@ -518,8 +513,7 @@ def fake_snapshot_download(repo_id, **kwargs): def test_restored_floating_owner_scale_matches_weight_dtype(): - # A floating (fp16/bf16) FP8 owner with no scale placeholder must get a buffer mirroring - # the weight dtype, not the checkpoint fp32 that would mismatch at matmul. #6749 + # A floating (fp16/bf16) FP8 owner with no scale placeholder must get a buffer mirroring the weight dtype, not checkpoint fp32. #6749 loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.bfloat16)) @@ -863,8 +857,7 @@ def test_restores_projection_expert_scale_on_projection_device(): assert restored == 1 assert skipped == 0 - # Scanner opens on CPU; the restored buffer must follow the CUDA projection, not the - # checkpoint tensor, or the expert forward hits a device mismatch. #6749 + # Scanner opens on CPU; the restored buffer must follow the CUDA projection, not the checkpoint tensor. #6749 assert model.expert.gate_up_proj_scale_inv.device.type == "cuda" @@ -1077,8 +1070,7 @@ def test_fastmodel_peft_base_mapping_clears_fp8_after_remap(): def test_fastmodel_peft_preserves_adapter_name_for_peft_load(): - # The FP8 base remap must compare against a pre-remap base name without reusing - # old_model_name, which still points at the adapter repo PeftModel loads. #6749 + # The FP8 base remap must compare against a pre-remap base name, not old_model_name (still the adapter repo). #6749 tree = ast.parse(LOADER.read_text()) offending = [] @@ -1200,8 +1192,7 @@ def test_offline_fp8_cache_forces_safetensors_loads(): ): hit_lines.add(child.lineno) - # Two loader paths, each forcing safetensors in both the offline-quantize branch and the - # mapped pre-quantized FP8 sibling branch: 4 assignments total. #6749 + # Two loader paths, each forcing safetensors in both the offline-quantize and mapped FP8 sibling branches: 4 total. #6749 assert len(hit_lines) == 4 @@ -1249,8 +1240,7 @@ def test_restored_scale_stays_a_parameter_when_placeholder_is_a_parameter(): loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) - # HF FP8 scales are Parameters; the restore must keep them Parameters, not demote them - # to buffers hidden from parameter inspection. + # HF FP8 scales are Parameters; the restore must keep them Parameters, not demote to buffers. model.fp8.weight_scale_inv = torch.nn.Parameter( torch.tensor([9.0], dtype = torch.float16), requires_grad = False ) @@ -1295,11 +1285,9 @@ def test_index_extra_patterns_cover_nonstandard_scale_shards(): json.dump(index, fh) extras = loader_utils._fp8_scale_index_extra_patterns(model_dir) - # Only the shard that actually holds a scale tensor is fetched; weight-only and non-scale - # shards are left out. + # Only the shard that actually holds a scale tensor is fetched; other shards are left out. assert extras == ["custom-shard-00001.safetensors"] - # The default snapshot globs do not match this non-standard shard name, so without the - # index-driven extra pass the scale shard would be under-fetched. + # The default globs miss this non-standard shard name, so without the index-driven pass the scale shard is under-fetched. default_patterns = loader_utils._fp8_scale_snapshot_allow_patterns() assert not any( fnmatch.fnmatch("custom-shard-00001.safetensors", pattern) for pattern in default_patterns @@ -1326,8 +1314,7 @@ def test_index_extra_patterns_prefix_subfolder(): def test_restores_transposed_column_wise_scale(): loader_utils = _load_loader_utils() model = torch.nn.Module() - # weight is [out=2, in=4]; a transposed/column-wise scale has shape[0] == in == 4, which - # exceeds out but is a layout the FP8 kernel supports and must not be skipped. + # weight is [out=2, in=4]; a transposed scale has shape[0] == in == 4, a layout the FP8 kernel supports and must not skip. model.fp8 = _Fp8Owner(weight = torch.randn(2, 4, dtype = torch.float16)) with tempfile.TemporaryDirectory() as checkpoint_dir: @@ -1406,8 +1393,7 @@ def test_variant_snapshot_patterns_are_index_and_single_file_only(): # First pass covers the single-file variant name and the variant index only. assert any(fnmatch.fnmatch("model.fp8.safetensors", p) for p in patterns) assert any(fnmatch.fnmatch("model.safetensors.index.fp8.json", p) for p in patterns) - # Shards come from the index-extra pass, so the shard glob is not in the first pass, and a - # stray alternate-component variant safetensors is never pulled. + # Shards come from the index-extra pass, so the shard glob is absent from the first pass and stray variant safetensors are never pulled. assert not any(fnmatch.fnmatch("model.fp8-00001-of-00002.safetensors", p) for p in patterns) assert not any(fnmatch.fnmatch("adapter.fp8.safetensors", p) for p in patterns) assert not any(fnmatch.fnmatch("vision_tower.fp8.safetensors", p) for p in patterns) @@ -1462,8 +1448,7 @@ def test_restores_static_activation_scale(): def test_restores_scale_when_placeholder_is_meta_parameter(): - # A meta Parameter placeholder cannot take `.data = ` (torch rejects - # meta -> concrete set_data); the restore must replace it instead of crashing. #6749 + # A meta Parameter placeholder rejects `.data = `; the restore must replace it instead of crashing. #6749 loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) @@ -1492,8 +1477,7 @@ def test_restores_scale_when_placeholder_is_meta_parameter(): def test_offline_quantize_preserves_bin_cache_on_failed_regeneration(monkeypatch, tmp_path): - # A failed re-quantization (offline / gated / removed source) must not delete a usable - # bin-only cache before the safetensors save has succeeded. #6749 + # A failed re-quantization must not delete a usable bin-only cache before the safetensors save succeeds. #6749 loader_utils = _load_loader_utils() import transformers @@ -1566,8 +1550,7 @@ def __init__(self, gate_up_proj): def test_restores_projection_scale_when_placeholder_is_meta(): - # A projection-only FP8 expert (materialized gate_up_proj, no weight) whose scale placeholder - # is still meta must restore onto the live projection device, not skip on the meta ref. #6749 + # A projection-only FP8 expert (no weight) with a meta scale placeholder must restore onto the live projection device, not skip. #6749 loader_utils = _load_loader_utils() model = torch.nn.Module() model.expert = _ProjFp8Owner(torch.randn(4, 4, dtype = torch.float16)) @@ -1592,8 +1575,7 @@ def test_restores_projection_scale_when_placeholder_is_meta(): def test_meta_parameter_replacement_preserves_metadata(): - # Replacing a meta Parameter placeholder must carry over custom attributes (e.g. block_size - # the fp8 kernels read) rather than dropping them. #6749 + # Replacing a meta Parameter placeholder must carry over custom attributes (e.g. block_size), not drop them. #6749 loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16)) @@ -1625,9 +1607,7 @@ def test_fp8_sibling_branch_forces_safetensors(): import re src = LOADER.read_text() - # Both pre-quantized FP8 sibling branches must force safetensors right after clearing - # load_in_fp8, so a caller-supplied use_safetensors=False cannot make from_pretrained look - # for absent .bin weights in a safetensors-only FP8 repo. #6749 + # Both pre-quantized FP8 sibling branches must force safetensors after clearing load_in_fp8, so use_safetensors=False cannot look for absent .bin. #6749 pattern = re.compile( r"restore_fp8_scales = True\s+load_in_fp8 = False\s+#[^\n]*\n(?:\s+#[^\n]*\n)*" r"\s+kwargs\[.use_safetensors.\] = True" @@ -1639,9 +1619,7 @@ def test_offline_fp8_bin_cache_fallback_not_forced_to_safetensors(): import re src = LOADER.read_text() - # The offline quantize helper may fall back to a preserved bin-only cache; forcing safetensors - # afterwards must be guarded by a check that the returned dir actually has safetensors, else - # the bin fallback is unloadable. Both loader paths must do this. #6749 + # Forcing safetensors after the offline quantize helper must be guarded by a real-safetensors check, else the bin fallback is unloadable. Both paths do this. #6749 pattern = re.compile( r"_offline_quantize_to_fp8\([^\n]*\)\s+restore_fp8_scales = True\s+#[^\n]*\n(?:\s+#[^\n]*\n)*" r'\s+if any\(name\.endswith\("\.safetensors"\)[^\n]*\n\s+kwargs\[.use_safetensors.\] = True' diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index d4d422c12fc0..93111b7ceb5a 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -505,9 +505,7 @@ def from_pretrained( ) model_name = _offline_quantize_to_fp8(model_name, fp8_mode, text_only = text_only) restore_fp8_scales = True - # The helper may fall back to a preserved bin-only cache when regeneration fails; - # only force safetensors when it actually produced one, else from_pretrained would - # look for absent safetensors in that bin cache. #6749 + # Force safetensors only if the helper produced one, not a bin-only fallback cache. #6749 if any(name.endswith(".safetensors") for name in os.listdir(model_name)): kwargs["use_safetensors"] = True else: @@ -518,9 +516,7 @@ def from_pretrained( if load_in_fp8 != False and new_model_name != old_model_name: restore_fp8_scales = True load_in_fp8 = False - # Pre-quantized FP8 siblings are safetensors-only, so force safetensors like - # the offline path above; otherwise a caller-supplied use_safetensors=False - # makes from_pretrained look for absent .bin weights and fail. #6749 + # Pre-quantized FP8 siblings are safetensors-only; force it so use_safetensors=False cannot fail on absent .bin. #6749 kwargs["use_safetensors"] = True # Check if pre-quantized models are allowed @@ -1213,9 +1209,7 @@ def from_pretrained( ) model_name = _offline_quantize_to_fp8(model_name, fp8_mode, text_only = text_only) restore_fp8_scales = True - # The helper may fall back to a preserved bin-only cache when regeneration fails; - # only force safetensors when it actually produced one, else from_pretrained would - # look for absent safetensors in that bin cache. #6749 + # Force safetensors only if the helper produced one, not a bin-only fallback cache. #6749 if any(name.endswith(".safetensors") for name in os.listdir(model_name)): kwargs["use_safetensors"] = True else: @@ -1226,9 +1220,7 @@ def from_pretrained( if load_in_fp8 != False and new_model_name != old_model_name: restore_fp8_scales = True load_in_fp8 = False - # Pre-quantized FP8 siblings are safetensors-only, so force safetensors like - # the offline path above; otherwise a caller-supplied use_safetensors=False - # makes from_pretrained look for absent .bin weights and fail. #6749 + # Pre-quantized FP8 siblings are safetensors-only; force it so use_safetensors=False cannot fail on absent .bin. #6749 kwargs["use_safetensors"] = True # Check if pre-quantized models are allowed @@ -1592,8 +1584,7 @@ def _dispatch_diffusion(): # Check base model again for PEFT model_name = peft_config.base_model_name_or_path if not use_exact_model_name: - # Use a separate local, not old_model_name, which still holds the adapter repo - # PeftModel.from_pretrained loads below; reusing it would attach the base. #6749 + # Separate local, not old_model_name (still the adapter repo), so the base is not reattached. #6749 base_before_remap = model_name model_name = get_model_name( model_name, @@ -1602,8 +1593,7 @@ def _dispatch_diffusion(): ) if load_in_fp8 != False and model_name != base_before_remap: load_in_fp8 = False - # Pre-quantized FP8 base siblings are safetensors-only; force safetensors so a - # caller-supplied use_safetensors=False does not fail on absent .bin weights. #6749 + # Pre-quantized FP8 base siblings are safetensors-only; force it so use_safetensors=False cannot fail on absent .bin. #6749 kwargs["use_safetensors"] = True # Check if pre-quantized models are allowed # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64) diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 4b574ede60e1..891b16fa9552 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -345,9 +345,8 @@ def _offline_quantize_to_fp8( config = text_config auto_model = AutoModelForImageTextToText if is_vlm else AutoModelForCausalLM auto_processor = AutoProcessor if is_vlm else AutoTokenizer - # Regenerate into a staging dir and only replace an existing (bin-only) cache once the - # safetensors save has succeeded, so a failed re-quantization (offline / gated / removed - # source model) leaves the previous loadable cache intact rather than deleting it. #6749 + # Stage the regen and swap in only after safetensors save succeeds, so a failed + # re-quantization leaves the previous loadable cache intact. #6749 staging_dir = new_model_name + ".fp8tmp" if os.path.isdir(staging_dir): shutil.rmtree(staging_dir) @@ -365,9 +364,8 @@ def _offline_quantize_to_fp8( gc.collect() tokenizer.save_pretrained(staging_dir) except Exception: - # Regeneration failed (offline / gated / removed source). If a previous bin-only - # cache is still on disk, fall back to it so an already-usable cache keeps loading - # after an upgrade instead of hard-failing. #6749 + # Regeneration failed: fall back to a previous bin-only cache if still on disk + # rather than hard-failing. #6749 if os.path.isdir(staging_dir): shutil.rmtree(staging_dir, ignore_errors = True) if needs_refresh and os.path.isdir(new_model_name): @@ -461,8 +459,7 @@ def _model_has_real_fp8_modules(model): ("gate_up_proj_scale_inv", ".gate_up_proj_scale_inv"), ("down_proj_scale", ".down_proj_scale"), ("down_proj_scale_inv", ".down_proj_scale_inv"), - # Static-activation FP8 checkpoints carry activation scales the expert forward reads when - # activation_scheme == "static"; restore them too so a dropped placeholder is repaired. #6749 + # Static-activation FP8 scales (activation_scheme == "static") the expert forward reads; restore too. #6749 ("activation_scale", ".activation_scale"), ("gate_proj_activation_scale", ".gate_proj_activation_scale"), ("up_proj_activation_scale", ".up_proj_activation_scale"), @@ -652,8 +649,7 @@ def _find_fp8_scale_inv_tensors( except Exception: return [] - # Fetch any scale shards the index maps to names outside the default globs; the first - # pass covers standard names, so standard repos add nothing here. + # Fetch scale shards the index maps outside the default globs; standard repos add nothing here. extra_patterns = _fp8_scale_index_extra_patterns(model_dir, subfolder, variant) missing = [ pattern @@ -794,10 +790,8 @@ def _restore_missing_fp8_weight_scale_inv( ): skipped += 1 continue - # Prefer a materialized device: a meta scale placeholder can sit next to an already - # loaded weight/projection (device-map/low-memory), so a live device is tried before any - # meta placeholder. A projection-only expert has no weight, so its live projection device - # is preferred over a meta reference too, else its scale would be skipped. #6749 + # Prefer a live (non-meta) reference/weight/projection device over a meta placeholder, + # else a projection-only expert's scale would be skipped. #6749 projection_device = _fp8_projection_device(module, attr_name) if isinstance(reference, torch.Tensor) and reference.device.type != "meta": target_device = reference.device @@ -832,8 +826,7 @@ def _restore_missing_fp8_weight_scale_inv( and scale_tensor.ndim > 0 and weight.ndim > 0 and scale_tensor.shape[0] > weight.shape[0] - # Keep transposed/column-wise scales (shape[0] == weight.shape[1]) the FP8 kernel - # handles; reject only scales matching neither weight rows nor columns. #6749 + # Keep transposed/column-wise scales (shape[0] == weight cols); reject only those matching neither rows nor cols. #6749 and (weight.ndim < 2 or scale_tensor.shape[0] != weight.shape[1]) ): skipped += 1 @@ -850,8 +843,7 @@ def _restore_missing_fp8_weight_scale_inv( and weight.is_floating_point() and not _has_float8_dtype(weight.dtype) ): - # Floating (fp16/bf16) FP8 owner with no reference scale: mirror the weight dtype - # instead of the checkpoint fp32. Packed/int8 and real float8 keep checkpoint dtype. #6749 + # Floating (fp16/bf16) FP8 owner, no reference scale: mirror weight dtype, not checkpoint fp32. #6749 try: restored_scale = restored_scale.to(dtype = weight.dtype) except Exception: @@ -859,19 +851,15 @@ def _restore_missing_fp8_weight_scale_inv( if attr_name in module._buffers: module._buffers[attr_name] = restored_scale elif attr_name in module._parameters: - # HF FP8 scales live in _parameters; keep them Parameters rather than demoting to a - # buffer, which would break tensor-parallel/optimizer/PEFT parameter inspection. #6749 + # HF FP8 scales live in _parameters; keep them Parameters (demoting to a buffer breaks TP/optimizer/PEFT). #6749 existing = module._parameters[attr_name] if isinstance(existing, torch.nn.Parameter) and existing.device.type != "meta": - # Update in place so runtime metadata (e.g. `block_size` fp8 dequant reads via - # getattr) survives; a fresh Parameter would drop it and force default geometry. #6749 + # Update in place so runtime metadata (e.g. `block_size`) survives; a fresh Parameter would drop it. #6749 with torch.no_grad(): existing.data = restored_scale else: - # A meta placeholder Parameter cannot take `.data = ` (torch rejects - # meta -> concrete set_data), so replace it to materialize the scale, copying the - # existing parameter's custom attributes (e.g. `block_size` the fp8 kernels read) - # so non-default block geometry is not lost. #6749 + # A meta placeholder Parameter rejects `.data = `, so replace it, + # copying custom attributes (e.g. `block_size`) so block geometry is not lost. #6749 new_param = torch.nn.Parameter( restored_scale, requires_grad = bool(getattr(existing, "requires_grad", False)), From da86965f00757f1b2c52f378f7bd53321c02a7b9 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Jul 2026 07:49:33 +0000 Subject: [PATCH 33/35] Restore FP8 scales onto the weight device over a CPU placeholder for PR #6749 --- tests/test_fp8_scale_inv_load.py | 25 +++++++++++++++++++++++++ unsloth/models/loader_utils.py | 11 ++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index be46c6c5ac75..9418dae8f8b5 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1637,3 +1637,28 @@ def test_peft_fp8_base_remap_forces_safetensors(): r"\s+kwargs\[.use_safetensors.\] = True" ) assert len(pattern.findall(src)) >= 1 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs cuda for cross-device test") +def test_prefers_weight_device_over_cpu_scale_placeholder(): + # A plain-tensor CPU scale placeholder alongside a CUDA weight (device_map does not move + # plain attributes) must restore onto the weight device, not the placeholder's CPU. #6749 + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16, device = "cuda")) + model.fp8.weight_scale_inv = torch.tensor([2.0], dtype = torch.float16) # plain CPU tensor + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.tensor([1.0], dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert model.fp8.weight_scale_inv.device.type == "cuda" diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 891b16fa9552..40060d8a36b2 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -790,15 +790,16 @@ def _restore_missing_fp8_weight_scale_inv( ): skipped += 1 continue - # Prefer a live (non-meta) reference/weight/projection device over a meta placeholder, - # else a projection-only expert's scale would be skipped. #6749 + # The scale must sit on the weight/projection compute device, so prefer that live device + # over a reference placeholder device_map may have left elsewhere (plain-tensor scale + # attributes are not moved). Fall back to the reference, then meta placeholders. #6749 projection_device = _fp8_projection_device(module, attr_name) - if isinstance(reference, torch.Tensor) and reference.device.type != "meta": - target_device = reference.device - elif isinstance(weight, torch.Tensor) and weight.device.type != "meta": + if isinstance(weight, torch.Tensor) and weight.device.type != "meta": target_device = weight.device elif projection_device is not None and projection_device.type != "meta": target_device = projection_device + elif isinstance(reference, torch.Tensor) and reference.device.type != "meta": + target_device = reference.device elif isinstance(reference, torch.Tensor): target_device = reference.device elif isinstance(weight, torch.Tensor): From 88985b57cdd076d7f3d6eef1cadd9c3b0c2ef739 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Jul 2026 08:25:39 +0000 Subject: [PATCH 34/35] Carry module block_size onto restored FP8 scales with dropped placeholders for PR #6749 --- tests/test_fp8_scale_inv_load.py | 24 ++++++++++++++++++++++++ unsloth/models/loader_utils.py | 10 ++++++++++ 2 files changed, 34 insertions(+) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 9418dae8f8b5..99c5a456c70c 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1662,3 +1662,27 @@ def test_prefers_weight_device_over_cpu_scale_placeholder(): assert restored == 1 assert skipped == 0 assert model.fp8.weight_scale_inv.device.type == "cuda" + + +def test_restored_scale_inherits_module_block_size_when_placeholder_dropped(): + # A dropped-placeholder scale registered as a bare buffer must carry the module's block_size, + # else the FP8 forward defaults to [128, 128] and mis-dequantizes non-default-block ckpts. #6749 + loader_utils = _load_loader_utils() + model = torch.nn.Module() + model.fp8 = _Fp8Owner(weight = torch.randn(256, 256, dtype = torch.float16)) + model.fp8.block_size = [64, 64] + + with tempfile.TemporaryDirectory() as checkpoint_dir: + _make_checkpoint( + checkpoint_dir, + {"fp8.weight_scale_inv": torch.randn(4, 4, dtype = torch.float32)}, + ) + restored, skipped = loader_utils._restore_missing_fp8_weight_scale_inv( + model, + model_name = checkpoint_dir, + local_files_only = True, + ) + + assert restored == 1 + assert skipped == 0 + assert getattr(model.fp8.weight_scale_inv, "block_size", None) == [64, 64] diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 40060d8a36b2..3ce82171592b 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -849,6 +849,16 @@ def _restore_missing_fp8_weight_scale_inv( restored_scale = restored_scale.to(dtype = weight.dtype) except Exception: pass + # Blockwise FP8 kernels read block geometry off the scale (getattr(weight_scale, + # "block_size", ...)); carry the module's block_size onto a restored scale whose + # placeholder was dropped, else the forward defaults to [128, 128] and mis-dequantizes + # non-default-block checkpoints. The _parameters branch below keeps the old attrs. #6749 + if not hasattr(restored_scale, "block_size"): + module_block_size = getattr(module, "block_size", None) + if module_block_size is None and isinstance(weight, torch.Tensor): + module_block_size = getattr(weight, "block_size", None) + if module_block_size is not None: + restored_scale.block_size = module_block_size if attr_name in module._buffers: module._buffers[attr_name] = restored_scale elif attr_name in module._parameters: From 720ebcea579ebb6cc795f535a497c5b272315c6e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Jul 2026 09:12:18 +0000 Subject: [PATCH 35/35] Tighten comments for PR #6749 --- tests/test_fp8_scale_inv_load.py | 6 ++---- unsloth/models/loader_utils.py | 9 ++------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/test_fp8_scale_inv_load.py b/tests/test_fp8_scale_inv_load.py index 99c5a456c70c..a51c37db5351 100644 --- a/tests/test_fp8_scale_inv_load.py +++ b/tests/test_fp8_scale_inv_load.py @@ -1641,8 +1641,7 @@ def test_peft_fp8_base_remap_forces_safetensors(): @pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs cuda for cross-device test") def test_prefers_weight_device_over_cpu_scale_placeholder(): - # A plain-tensor CPU scale placeholder alongside a CUDA weight (device_map does not move - # plain attributes) must restore onto the weight device, not the placeholder's CPU. #6749 + # A plain-tensor CPU scale placeholder alongside a CUDA weight must restore onto the weight device, not CPU. #6749 loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _Fp8Owner(weight = torch.randn(4, 4, dtype = torch.float16, device = "cuda")) @@ -1665,8 +1664,7 @@ def test_prefers_weight_device_over_cpu_scale_placeholder(): def test_restored_scale_inherits_module_block_size_when_placeholder_dropped(): - # A dropped-placeholder scale registered as a bare buffer must carry the module's block_size, - # else the FP8 forward defaults to [128, 128] and mis-dequantizes non-default-block ckpts. #6749 + # A dropped-placeholder scale must carry the module's block_size, else the FP8 forward defaults to [128, 128] and mis-dequantizes non-default-block ckpts. #6749 loader_utils = _load_loader_utils() model = torch.nn.Module() model.fp8 = _Fp8Owner(weight = torch.randn(256, 256, dtype = torch.float16)) diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 3ce82171592b..8b1e39cf4d29 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -790,9 +790,7 @@ def _restore_missing_fp8_weight_scale_inv( ): skipped += 1 continue - # The scale must sit on the weight/projection compute device, so prefer that live device - # over a reference placeholder device_map may have left elsewhere (plain-tensor scale - # attributes are not moved). Fall back to the reference, then meta placeholders. #6749 + # Prefer the live weight/projection compute device over a reference placeholder; fall back to reference, then meta. #6749 projection_device = _fp8_projection_device(module, attr_name) if isinstance(weight, torch.Tensor) and weight.device.type != "meta": target_device = weight.device @@ -849,10 +847,7 @@ def _restore_missing_fp8_weight_scale_inv( restored_scale = restored_scale.to(dtype = weight.dtype) except Exception: pass - # Blockwise FP8 kernels read block geometry off the scale (getattr(weight_scale, - # "block_size", ...)); carry the module's block_size onto a restored scale whose - # placeholder was dropped, else the forward defaults to [128, 128] and mis-dequantizes - # non-default-block checkpoints. The _parameters branch below keeps the old attrs. #6749 + # Carry the module's block_size onto a dropped-placeholder scale, else the blockwise forward defaults to [128, 128] and mis-dequantizes non-default-block checkpoints. #6749 if not hasattr(restored_scale, "block_size"): module_block_size = getattr(module, "block_size", None) if module_block_size is None and isinstance(weight, torch.Tensor):